diff --git a/.gitignore b/.gitignore index e6e327327c2a72bc2262d9b2923314c950c78cf6..1ff11080569e9312369f6e9c00463e25853fd38b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,6 @@ tests/hdfs/ nmake/ sln/ hdfs/ -c/ taoshebei/ taosdalipu/ Target/ @@ -79,3 +78,15 @@ tests/comparisonTest/opentsdb/opentsdbtest/.settings/ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ + +# Emacs +# -*- mode: gitignore; -*- +*~ +\#*\# +/.emacs.desktop +/.emacs.desktop.lock +*.elc +auto-save-list +tramp +.\#* +TAGS diff --git a/.gitmodules b/.gitmodules index 156226d54486c17e64b9c514e47e3a7dc3fe6942..74afbbf9973abec6423633b848181b349de4ed6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,9 @@ [submodule "src/connector/grafanaplugin"] path = src/connector/grafanaplugin url = https://github.com/taosdata/grafanaplugin +[submodule "src/connector/hivemq-tdengine-extension"] + path = src/connector/hivemq-tdengine-extension + url = https://github.com/huskar-t/hivemq-tdengine-extension.git [submodule "tests/examples/rust"] path = tests/examples/rust url = https://github.com/songtianyi/tdengine-rust-bindings.git -[submodule "src/connector/hivemq-tdengine-extension"] - path = src/connector/hivemq-tdengine-extension - url = https://github.com/huskar-t/hivemq-tdengine-extension.git \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 9aac6c597cd9608c293b3d0f9b49c0db3fa5d9a4..0617d759768251fcf09aa2316f3556f20c1718d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -171,6 +171,8 @@ matrix: - build-essential - cmake - binutils-2.26 + - unixodbc + - unixodbc-dev env: - DESC="trusty/gcc-4.8/bintuils-2.26 build" @@ -198,6 +200,8 @@ matrix: packages: - build-essential - cmake + - unixodbc + - unixodbc-dev before_script: - export TZ=Asia/Harbin @@ -252,6 +256,8 @@ matrix: packages: - build-essential - cmake + - unixodbc + - unixodbc-dev env: - DESC="arm64 xenial build" @@ -270,19 +276,21 @@ matrix: fi - make > /dev/null -# - os: osx -# language: c -# compiler: clang -# env: DESC="mac/clang build" -# git: -# - depth: 1 -# addons: -# homebrew: -# - cmake -# -# script: -# - cd ${TRAVIS_BUILD_DIR} -# - mkdir debug -# - cd debug -# - cmake .. > /dev/null -# - make > /dev/null + - os: osx + osx_image: xcode11.4 + language: c + compiler: clang + env: DESC="mac/clang build" + git: + - depth: 1 + addons: + homebrew: + - cmake + - unixodbc + + script: + - cd ${TRAVIS_BUILD_DIR} + - mkdir debug + - cd debug + - cmake .. > /dev/null + - make > /dev/null diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c474a355dc68e7ef3b1fcba0588de1060487540..e0d6e82923ce66abccefda6ee685ec98d6450d2e 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) IF (CMAKE_VERSION VERSION_LESS 3.0) PROJECT(TDengine CXX) SET(PROJECT_VERSION_MAJOR "${LIB_MAJOR_VERSION}") @@ -13,8 +13,11 @@ ENDIF () SET(TD_ACCOUNT FALSE) SET(TD_ADMIN FALSE) SET(TD_GRANT FALSE) -SET(TD_MQTT TRUE) +SET(TD_MQTT FALSE) SET(TD_TSDB_PLUGINS FALSE) +SET(TD_STORAGE FALSE) +SET(TD_TOPIC FALSE) +SET(TD_MODULE FALSE) SET(TD_COVER FALSE) SET(TD_MEM_CHECK FALSE) @@ -29,6 +32,11 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) + +IF (TD_WINDOWS OR TD_DARWIN) + SET(TD_SOMODE_STATIC TRUE) +ENDIF () + INCLUDE(cmake/define.inc) INCLUDE(cmake/env.inc) INCLUDE(cmake/version.inc) diff --git a/Jenkinsfile b/Jenkinsfile index ebac32cb241af1a35556262690544f84ca94d9fc..bac0cce33b9b645373170247fb91df31fad6f646 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,6 +5,7 @@ node { git url: 'https://github.com/taosdata/TDengine.git' } +def skipstage=0 def abortPreviousBuilds() { def currentJobName = env.JOB_NAME @@ -24,7 +25,7 @@ def abortPreviousBuilds() { build.doKill() //doTerm(),doKill(),doTerm() } } -//abort previous build +// abort previous build abortPreviousBuilds() def abort_previous(){ def buildNumber = env.BUILD_NUMBER as int @@ -32,30 +33,29 @@ def abort_previous(){ milestone(buildNumber) } def pre_test(){ - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - sudo rmtaos - ''' - } - sh ''' + + sh ''' + sudo rmtaos || echo "taosd has not installed" + ''' + sh ''' + killall -9 taosd ||echo "no taosd running" + killall -9 gdb || echo "no gdb running" cd ${WKC} + git reset --hard HEAD~10 >/dev/null git checkout develop - git reset --hard HEAD~10 - git pull - git fetch - git checkout ${CHANGE_BRANCH} - git reset --hard HEAD~10 - git pull - git merge develop + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git clean -dfx cd ${WK} git reset --hard HEAD~10 - git checkout develop - git pull + git checkout develop + git pull >/dev/null cd ${WK} export TZ=Asia/Harbin date - rm -rf ${WK}/debug + git clean -dfx mkdir debug cd debug cmake .. > /dev/null @@ -65,6 +65,7 @@ def pre_test(){ ''' return 1 } + pipeline { agent none @@ -74,55 +75,101 @@ pipeline { } stages { - + stage('pre_build'){ + agent{label 'master'} + when { + changeRequest() + } + steps { + script{ + abort_previous() + abortPreviousBuilds() + } + sh''' + cp -r ${WORKSPACE} ${WORKSPACE}.tes + cd ${WORKSPACE}.tes + git checkout develop + git pull + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + ''' + script{ + env.skipstage=sh(script:"cd ${WORKSPACE}.tes && git --no-pager diff --name-only FETCH_HEAD develop|grep -v -E '.*md|//src//connector|Jenkinsfile|test-all.sh' || echo 0 ",returnStdout:true) + } + println env.skipstage + sh''' + rm -rf ${WORKSPACE}.tes + ''' + } + } stage('Parallel test stage') { //only build pr when { changeRequest() + expression { + env.skipstage != 0 + } } parallel { - stage('python_1') { + stage('python_1_s1') { agent{label 'p1'} steps { pre_test() - sh ''' - cd ${WKC}/tests - find pytest -name '*'sql|xargs rm -rf - ./test-all.sh p1 - date''' + timeout(time: 45, unit: 'MINUTES'){ + sh ''' + date + cd ${WKC}/tests + ./test-all.sh p1 + date''' + } + } } - stage('python_2') { + stage('python_2_s5') { agent{label 'p2'} steps { pre_test() - sh ''' - cd ${WKC}/tests - find pytest -name '*'sql|xargs rm -rf - ./test-all.sh p2 - date''' - sh ''' - cd ${WKC}/tests - ./test-all.sh b4fq - ''' + timeout(time: 45, unit: 'MINUTES'){ + sh ''' + date + cd ${WKC}/tests + ./test-all.sh p2 + date''' + } + } + } + stage('python_3_s6') { + agent{label 'p3'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh p3 + date''' + } } } - stage('test_b1') { + stage('test_b1_s2') { agent{label 'b1'} - steps { - pre_test() - sh ''' - cd ${WKC}/tests - ./test-all.sh b1fq - date''' + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + cd ${WKC}/tests + ./test-all.sh b1fq + date''' + } } } - stage('test_crash_gen') { + stage('test_crash_gen_s3') { agent{label "b2"} + steps { pre_test() catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { @@ -131,22 +178,34 @@ pipeline { ./crash_gen.sh -a -p -t 4 -s 2000 ''' } + + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_crash_gen_val_log.sh + ''' catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/pytest - ./handle_crash_gen_val_log.sh + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_taosd_val_log.sh ''' } - sh ''' - date - cd ${WKC}/tests - ./test-all.sh b2fq - date - ''' + timeout(time: 45, unit: 'MINUTES'){ + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b2fq + date + ''' + } + } } - stage('test_valgrind') { + stage('test_valgrind_s4') { agent{label "b3"} steps { @@ -157,18 +216,159 @@ pipeline { ./valgrind-test.sh 2>&1 > mem-error-out.log ./handle_val_log.sh ''' - } - sh ''' - date - cd ${WKC}/tests - ./test-all.sh b3fq - date''' + } + timeout(time: 45, unit: 'MINUTES'){ + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b3fq + date''' + sh ''' + date + cd ${WKC}/tests + ./test-all.sh full example + date''' + } } } - - + stage('test_b4_s7') { + agent{label 'b4'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b4fq + cd ${WKC}/tests + ./test-all.sh p4 + cd ${WKC}/tests + ./test-all.sh full jdbc + cd ${WKC}/tests + ./test-all.sh full unit + date''' + } + } + } + stage('test_b5_s8') { + agent{label 'b5'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b5fq + date''' + } + } + } + stage('test_b6_s9') { + agent{label 'b6'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b6fq + date''' + } + } + } + stage('test_b7_s10') { + agent{label 'b7'} + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + date + cd ${WKC}/tests + ./test-all.sh b7fq + date''' + } + } + } } } } + post { + success { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", + body: """ + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支:${env.BRANCH_NAME}
  • +
  • 构建结果: Successful
  • +
  • 构建编号:${BUILD_NUMBER}
  • +
  • 触发用户:${env.CHANGE_AUTHOR}
  • +
  • 提交信息:${env.CHANGE_TITLE}
  • +
  • 构建地址:${BUILD_URL}
  • +
  • 构建日志:${BUILD_URL}console
  • + +
    +
+
+ + """, + to: "${env.CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' FAIL", + body: """ + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支:${env.BRANCH_NAME}
  • +
  • 构建结果: Failure
  • +
  • 构建编号:${BUILD_NUMBER}
  • +
  • 触发用户:${env.CHANGE_AUTHOR}
  • +
  • 提交信息:${env.CHANGE_TITLE}
  • +
  • 构建地址:${BUILD_URL}
  • +
  • 构建日志:${BUILD_URL}console
  • + +
    +
+
+ + """, + to: "${env.CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + } } diff --git a/README-CN.md b/README-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..d4c10e71d684ab5d21c1c767c398707956946232 --- /dev/null +++ b/README-CN.md @@ -0,0 +1,273 @@ +[![Build Status](https://travis-ci.org/taosdata/TDengine.svg?branch=master)](https://travis-ci.org/taosdata/TDengine) +[![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) +[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) +[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) +[![tdengine](https://snapcraft.io//tdengine/badge.svg)](https://snapcraft.io/tdengine) + +[![TDengine](TDenginelogo.png)](https://www.taosdata.com) + +简体中文 | [English](./README.md) + +# TDengine 简介 + +TDengine是涛思数据专为物联网、车联网、工业互联网、IT运维等设计和优化的大数据平台。除核心的快10倍以上的时序数据库功能外,还提供缓存、数据订阅、流式计算等功能,最大程度减少研发和运维的复杂度,且核心代码,包括集群功能全部开源(开源协议,AGPL v3.0)。 + +- 10 倍以上性能提升。定义了创新的数据存储结构,单核每秒就能处理至少2万次请求,插入数百万个数据点,读出一千万以上数据点,比现有通用数据库快了十倍以上。 +- 硬件或云服务成本降至1/5。由于超强性能,计算资源不到通用大数据方案的1/5;通过列式存储和先进的压缩算法,存储空间不到通用数据库的1/10。 +- 全栈时序数据处理引擎。将数据库、消息队列、缓存、流式计算等功能融合一起,应用无需再集成Kafka/Redis/HBase/Spark等软件,大幅降低应用开发和维护成本。 +- 强大的分析功能。无论是十年前还是一秒钟前的数据,指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell/Python/R/Matlab随时进行。 +- 与第三方工具无缝连接。不用一行代码,即可与Telegraf, Grafana, EMQ X, Prometheus, Matlab, R集成。后续还将支持MQTT, OPC, Hadoop,Spark等, BI工具也将无缝连接。 +- 零运维成本、零学习成本。安装、集群一秒搞定,无需分库分表,实时备份。标准SQL,支持JDBC,RESTful,支持Python/Java/C/C++/Go/Node.JS, 与MySQL相似,零学习成本。 + +# 文档 + +TDengine是一个高效的存储、查询、分析时序大数据的平台,专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它,但建议您在使用前仔细阅读一遍下面的文档,特别是 [数据模型](https://www.taosdata.com/cn/documentation/architecture) 与 [数据建模](https://www.taosdata.com/cn/documentation/model)。除本文档之外,欢迎 [下载产品白皮书](https://www.taosdata.com/downloads/TDengine%20White%20Paper.pdf)。 + +# 生成 + +TDengine目前2.0版服务器仅能在Linux系统上安装和运行,后续会支持Windows、macOS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64,后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。本快速指南仅适用于通过源码安装。 + +## 安装工具 + +### Ubuntu 16.04 及以上版本 & Debian: + +```bash +sudo apt-get install -y gcc cmake build-essential git +``` + +### Ubuntu 14.04: + +```bash +sudo apt-get install -y gcc cmake3 build-essential git binutils-2.26 +export PATH=/usr/lib/binutils-2.26/bin:$PATH +``` + +编译或打包 JDBC 驱动源码,需安装 Java JDK 8 或以上版本和 Apache Maven 2.7 或以上版本。 + +安装 OpenJDK 8: + +```bash +sudo apt-get install -y openjdk-8-jdk +``` + +安装 Apache Maven: + +```bash +sudo apt-get install -y maven +``` + +### CentOS 7: + +```bash +sudo yum install -y gcc gcc-c++ make cmake git +``` + +安装 OpenJDK 8: + +```bash +sudo yum install -y java-1.8.0-openjdk +``` + +安装 Apache Maven: + +```bash +sudo yum install -y maven +``` + +### CentOS 8 & Fedora: + +```bash +sudo dnf install -y gcc gcc-c++ make cmake epel-release git +``` + +安装 OpenJDK 8: + +```bash +sudo dnf install -y java-1.8.0-openjdk +``` + +安装 Apache Maven: + +```bash +sudo dnf install -y maven +``` + +## 获取源码 + +首先,你需要从 GitHub 克隆源码: + +```bash +git clone https://github.com/taosdata/TDengine.git +cd TDengine +``` + +Go 连接器和 Grafana 插件在其他独立仓库,如果安装它们的话,需要在 TDengine 目录下通过此命令安装: + +```bash +git submodule update --init --recursive +``` + +## 生成 TDengine + +### Linux 系统 + +```bash +mkdir debug && cd debug +cmake .. && cmake --build . +``` + +在X86-64、X86、arm64 和 arm32 平台上,TDengine 生成脚本可以自动检测机器架构。也可以手动配置 CPUTYPE 参数来指定 CPU 类型,如 aarch64 或 aarch32 等。 + +aarch64: + +```bash +cmake .. -DCPUTYPE=aarch64 && cmake --build . +``` + +aarch32: + +```bash +cmake .. -DCPUTYPE=aarch32 && cmake --build . +``` + +### Windows 系统 + +如果你使用的是 Visual Studio 2013 版本: + +打开 cmd.exe,执行 vcvarsall.bat 时,为 64 位操作系统指定“x86_amd64”,为 32 位操作系统指定“x86”。 + +```bash +mkdir debug && cd debug +"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < x86_amd64 | x86 > +cmake .. -G "NMake Makefiles" +nmake +``` + +如果你使用的是 Visual Studio 2019 或 2017 版本: + +打开cmd.exe,执行 vcvarsall.bat 时,为 64 位操作系统指定“x64”,为 32 位操作系统指定“x86”。 + +```bash +mkdir debug && cd debug +"c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > +cmake .. -G "NMake Makefiles" +nmake +``` + +你也可以从开始菜单中找到"Visual Studio < 2019 | 2017 >"菜单项,根据你的系统选择"x64 Native Tools Command Prompt for VS < 2019 | 2017 >"或"x86 Native Tools Command Prompt for VS < 2019 | 2017 >",打开命令行窗口,执行: + +```bash +mkdir debug && cd debug +cmake .. -G "NMake Makefiles" +nmake +``` + +### Mac OS X 系统 + +安装 Xcode 命令行工具和 cmake. 在 Catalina 和 Big Sur 操作系统上,需要安装 XCode 11.4+ 版本。 + +```bash +mkdir debug && cd debug +cmake .. && cmake --build . +``` + +# 安装 + +如果你不想安装,可以直接在shell中运行。生成完成后,安装 TDengine: +```bash +make install +``` + +用户可以在[文件目录结构](https://www.taosdata.com/cn/documentation/administrator#directories)中了解更多在操作系统中生成的目录或文件。 + +安装成功后,在终端中启动 TDengine 服务: + +```bash +taosd +``` + +用户可以使用 TDengine Shell 来连接 TDengine 服务,在终端中,输入: + +```bash +taos +``` + +如果 TDengine Shell 连接服务成功,将会打印出欢迎消息和版本信息。如果失败,则会打印出错误消息。 + +## 快速运行 + +TDengine 生成后,在终端执行以下命令: + +```bash +./build/bin/taosd -c test/cfg +``` + +在另一个终端,使用 TDengine shell 连接服务器: + +```bash +./build/bin/taos -c test/cfg +``` + +"-c test/cfg"指定系统配置文件所在目录。 + +# 体验 TDengine + +在TDengine终端中,用户可以通过SQL命令来创建/删除数据库、表等,并进行插入查询操作。 + +```bash +create database demo; +use demo; +create table t (ts timestamp, speed int); +insert into t values ('2019-07-15 00:00:00', 10); +insert into t values ('2019-07-15 01:00:00', 20); +select * from t; + ts | speed | +=================================== + 19-07-15 00:00:00.000| 10| + 19-07-15 01:00:00.000| 20| +Query OK, 2 row(s) in set (0.001700s) +``` + +# 应用开发 + +## 官方连接器 + +TDengine 提供了丰富的应用程序开发接口,其中包括C/C++、Java、Python、Go、Node.js、C# 、RESTful 等,便于用户快速开发应用: + +- Java + +- C/C++ + +- Python + +- Go + +- RESTful API + +- Node.js + +## 第三方连接器 + +TDengine 社区生态中也有一些非常友好的第三方连接器,可以通过以下链接访问它们的源码。 + +- [Rust Connector](https://github.com/taosdata/TDengine/tree/master/tests/examples/rust) +- [.Net Core Connector](https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos) +- [Lua Connector](https://github.com/taosdata/TDengine/tree/develop/tests/examples/lua) + +# 运行和添加测试例 + +TDengine 的测试框架和所有测试例全部开源。 + +点击 [这里](tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md),了解如何运行测试例和添加新的测试例。 + +# 成为社区贡献者 + +点击 [这里](https://www.taosdata.com/cn/contributor/),了解如何成为 TDengine 的贡献者。 + +# 加入技术交流群 + +TDengine 官方社群「物联网大数据群」对外开放,欢迎您加入讨论。搜索微信号 "tdengine",加小T为好友,即可入群。 + +# [谁在使用TDengine](https://github.com/taosdata/TDengine/issues/2432) + +欢迎所有 TDengine 用户及贡献者在 [这里](https://github.com/taosdata/TDengine/issues/2432) 分享您在当前工作中开发/使用 TDengine 的故事。 diff --git a/README.md b/README.md index 566b3e5c8bdf70e06bf13af644b21260b5f87164..45a955f458c953af3e4135430d6cce3f56017c1a 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ [![TDengine](TDenginelogo.png)](https://www.taosdata.com) +English | [简体中文](./README-CN.md) + # What is TDengine? TDengine is an open-sourced big data platform under [GNU AGPL v3.0](http://www.gnu.org/licenses/agpl-3.0.html), designed and optimized for the Internet of Things (IoT), Connected Cars, Industrial IoT, and IT Infrastructure and Application Monitoring. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and cost of development and operation. @@ -29,7 +31,7 @@ For user manual, system design and architecture, engineering blogs, refer to [TD # Building At the moment, TDengine only supports building and running on Linux systems. You can choose to [install from packages](https://www.taosdata.com/en/getting-started/#Install-from-Package) or from the source code. This quick guide is for installation from the source only. -To build TDengine, use [CMake](https://cmake.org/) 3.5 or higher versions in the project directory. +To build TDengine, use [CMake](https://cmake.org/) 2.8.12.x or higher versions in the project directory. ## Install tools @@ -57,9 +59,7 @@ sudo apt-get install -y maven ### Centos 7: ```bash -sudo yum install -y gcc gcc-c++ make cmake3 epel-release git -sudo yum remove -y cmake -sudo ln -s /usr/bin/cmake3 /usr/bin/cmake +sudo yum install -y gcc gcc-c++ make cmake git ``` To install openjdk-8: @@ -110,7 +110,8 @@ mkdir debug && cd debug cmake .. && cmake --build . ``` -To compile on an ARM processor (aarch64 or aarch32), please add option CPUTYPE as below: +TDengine build script can detect the host machine's architecture on X86-64, X86, arm64 and arm32 platform. +You can also specify CPUTYPE option like aarch64 or aarch32 too if the detection result is not correct: aarch64: ```bash @@ -126,61 +127,77 @@ cmake .. -DCPUTYPE=aarch32 && cmake --build . If you use the Visual Studio 2013, please open a command window by executing "cmd.exe". Please specify "x86_amd64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. -``` +```cmd mkdir debug && cd debug "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < x86_amd64 | x86 > cmake .. -G "NMake Makefiles" nmake ``` -If you use the Visual Studio 2019, please open a command window by executing "cmd.exe". +If you use the Visual Studio 2019 or 2017: + +please open a command window by executing "cmd.exe". Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat. -``` + +```cmd mkdir debug && cd debug "c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 > cmake .. -G "NMake Makefiles" nmake ``` -Or, you can open a command window by clicking Visual Studio 2019 menu "Tools -> Command Line -> Developer Command Prompt" or "Tools -> Command Line -> Developer PowerShell" then execute commands as follows: -``` +Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows: +```cmd mkdir debug && cd debug cmake .. -G "NMake Makefiles" nmake ``` -# Quick Run -# Quick Run -To quickly start a TDengine server after building, run the command below in terminal: -```cmd -./build/bin/taosd -c test/cfg -``` -In another terminal, use the TDengine shell to connect the server: -``` -./build/bin/taos -c test/cfg +### On Mac OS X platform + +Please install XCode command line tools and cmake. Verified with XCode 11.4+ on Catalina and Big Sur. + +```shell +mkdir debug && cd debug +cmake .. && cmake --build . ``` -option "-c test/cfg" specifies the system configuration file directory. # Installing + After building successfully, TDengine can be installed by: -```cmd -make install +```bash +sudo make install ``` -Users can find more information about directories installed on the system in the [directory and files](https://www.taosdata.com/en/documentation/administrator/#Directory-and-Files) section. It should be noted that installing from source code does not configure service management for TDengine. + +Users can find more information about directories installed on the system in the [directory and files](https://www.taosdata.com/en/documentation/administrator/#Directory-and-Files) section. Since version 2.0, installing from source code will also configure service management for TDengine. Users can also choose to [install from packages](https://www.taosdata.com/en/getting-started/#Install-from-Package) for it. To start the service after installation, in a terminal, use: -```cmd -taosd +```bash +sudo systemctl start taosd ``` Then users can use the [TDengine shell](https://www.taosdata.com/en/getting-started/#TDengine-Shell) to connect the TDengine server. In a terminal, use: -```cmd +```bash taos ``` If TDengine shell connects the server successfully, welcome messages and version info are printed. Otherwise, an error message is shown. +## Quick Run + +If you don't want to run TDengine as a service, you can run it in current shell. For example, to quickly start a TDengine server after building, run the command below in terminal: +```bash +./build/bin/taosd -c test/cfg +``` + +In another terminal, use the TDengine shell to connect the server: +```bash +./build/bin/taos -c test/cfg +``` + +option "-c test/cfg" specifies the system configuration file directory. + # Try TDengine It is easy to run SQL commands from TDengine shell which is the same as other SQL databases. ```sql @@ -233,3 +250,6 @@ Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to th Add WeChat “tdengine” to join the group,you can communicate with other users. +# [User List](https://github.com/taosdata/TDengine/issues/2432) + +If you are using TDengine and feel it helps or you'd like to do some contributions, please add your company to [user list](https://github.com/taosdata/TDengine/issues/2432) and let us know your needs. diff --git a/cmake/define.inc b/cmake/define.inc index 5d4d94ff42c73a7ca11f32f8318526bfcfe4e525..6f49630d5c25d0f1519f3e6c606fe28a026add2d 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -21,6 +21,18 @@ IF (TD_TSDB_PLUGINS) ADD_DEFINITIONS(-D_TSDB_PLUGINS) ENDIF () +IF (TD_STORAGE) + ADD_DEFINITIONS(-D_STORAGE) +ENDIF () + +IF (TD_TOPIC) + ADD_DEFINITIONS(-D_TOPIC) +ENDIF () + +IF (TD_MODULE) + ADD_DEFINITIONS(-D_MODULE) +ENDIF () + IF (TD_GODLL) ADD_DEFINITIONS(-D_TD_GO_DLL_) ENDIF () @@ -128,6 +140,8 @@ IF (TD_DARWIN_64) SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") SET(DEBUG_FLAGS "-O0 -g3 -DDEBUG") SET(RELEASE_FLAGS "-Og") + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) ENDIF () IF (TD_WINDOWS) @@ -139,6 +153,9 @@ IF (TD_WINDOWS) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) IF (NOT TD_GODLL) SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) + SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") + ENDIF () SET(DEBUG_FLAGS "/Zi /W3 /GL") SET(RELEASE_FLAGS "/W0 /O3 /GL") ENDIF () diff --git a/cmake/input.inc b/cmake/input.inc index e8324887a051ae6ac27eff6b748c824d1f0a3fa5..00e0e1bc0f00fd8ba8e679f93d5f75e1b75e6bcd 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -9,6 +9,22 @@ ELSEIF (${ACCOUNT} MATCHES "false") MESSAGE(STATUS "Build without account plugins") ENDIF () +IF (${TOPIC} MATCHES "true") + SET(TD_TOPIC TRUE) + MESSAGE(STATUS "Build with topic plugins") +ELSEIF (${TOPIC} MATCHES "false") + SET(TD_TOPIC FALSE) + MESSAGE(STATUS "Build without topic plugins") +ENDIF () + +IF (${TD_MODULE} MATCHES "true") + SET(TD_MODULE TRUE) + MESSAGE(STATUS "Build with module plugins") +ELSEIF (${TOPIC} MATCHES "false") + SET(TD_MODULE FALSE) + MESSAGE(STATUS "Build without module plugins") +ENDIF () + IF (${COVER} MATCHES "true") SET(TD_COVER TRUE) MESSAGE(STATUS "Build with test coverage") diff --git a/cmake/install.inc b/cmake/install.inc index 55b3fa188ff2479b245213ef5dab15cfa2909f02..5823ef743ed5b5b57f3915a82aa9e1e21f850d2f 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.15-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.25-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/cmake/platform.inc b/cmake/platform.inc index 889f6c73cf2be982df7f1db207114e58d6af3e92..dcd0183e27b58f2ebd22c584fab359e025044cba 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -78,29 +78,58 @@ ELSE() EXIT () ENDIF () -# if generate ARM version: -# cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 -IF (${CPUTYPE} MATCHES "aarch32") - SET(TD_LINUX TRUE) - SET(TD_LINUX_32 FALSE) - SET(TD_ARM_32 TRUE) - MESSAGE(STATUS "input cpuType: aarch32") -ELSEIF (${CPUTYPE} MATCHES "aarch64") - SET(TD_LINUX TRUE) - SET(TD_LINUX_64 FALSE) - SET(TD_ARM_64 TRUE) - MESSAGE(STATUS "input cpuType: aarch64") -ELSEIF (${CPUTYPE} MATCHES "mips64") - SET(TD_LINUX TRUE) - SET(TD_LINUX_64 FALSE) - SET(TD_MIPS_64 TRUE) - MESSAGE(STATUS "input cpuType: mips64") -ELSEIF (${CPUTYPE} MATCHES "x64") - MESSAGE(STATUS "input cpuType: x64") -ELSEIF (${CPUTYPE} MATCHES "x86") - MESSAGE(STATUS "input cpuType: x86") +IF ("${CPUTYPE}" STREQUAL "") + MESSAGE(STATUS "The current platform " ${CMAKE_SYSTEM_PROCESSOR} " is detected") + + IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") + MESSAGE(STATUS "The current platform is amd64") + MESSAGE(STATUS "Set CPUTYPE to x64") + SET(CPUTYPE "x64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") + MESSAGE(STATUS "The current platform is x86") + MESSAGE(STATUS "Set CPUTYPE to x86") + SET(CPUTYPE "x32") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") + MESSAGE(STATUS "Set CPUTYPE to aarch32") + SET(CPUTYPE "aarch32") + MESSAGE(STATUS "Set CPUTYPE to aarch32") + SET(TD_LINUX TRUE) + SET(TD_LINUX_32 FALSE) + SET(TD_ARM_32 TRUE) + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + SET(CPUTYPE "aarch64") + MESSAGE(STATUS "Set CPUTYPE to aarch64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_ARM_64 TRUE) + ENDIF () + ELSE () - MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) + # if generate ARM version: + # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 + IF (${CPUTYPE} MATCHES "aarch32") + SET(TD_LINUX TRUE) + SET(TD_LINUX_32 FALSE) + SET(TD_ARM_32 TRUE) + MESSAGE(STATUS "input cpuType: aarch32") + ELSEIF (${CPUTYPE} MATCHES "aarch64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_ARM_64 TRUE) + MESSAGE(STATUS "input cpuType: aarch64") + ELSEIF (${CPUTYPE} MATCHES "mips64") + SET(TD_LINUX TRUE) + SET(TD_LINUX_64 FALSE) + SET(TD_MIPS_64 TRUE) + MESSAGE(STATUS "input cpuType: mips64") + ELSEIF (${CPUTYPE} MATCHES "x64") + MESSAGE(STATUS "input cpuType: x64") + ELSEIF (${CPUTYPE} MATCHES "x86") + MESSAGE(STATUS "input cpuType: x86") + ELSE () + MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) + ENDIF () + ENDIF () # cmake -DOSTYPE=Ningsi diff --git a/cmake/version.inc b/cmake/version.inc old mode 100644 new mode 100755 index 0509a5ce1b2d44f4986c734060853b3aa32addac..b1e09c9532ef8193b32c99180d35946cc6ef5c7e --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.14.0") + SET(TD_VER_NUMBER "2.0.18.0") ENDIF () IF (DEFINED VERCOMPATIBLE) @@ -13,16 +13,40 @@ ELSE () SET(TD_VER_COMPATIBLE "2.0.0.0") ENDIF () +find_program(HAVE_GIT NAMES git) + IF (DEFINED GITINFO) SET(TD_VER_GIT ${GITINFO}) +ELSEIF (HAVE_GIT) + execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${TD_COMMUNITY_DIR} OUTPUT_VARIABLE GIT_COMMITID) + message(STATUS "git log result:${GIT_COMMITID}") + IF (GIT_COMMITID) + string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID}) + SET(TD_VER_GIT ${GIT_COMMITID}) + ELSE () + message(STATUS "not a git repository") + SET(TD_VER_GIT "no git commit id") + ENDIF () ELSE () - SET(TD_VER_GIT "community") + message(STATUS "no git cmd") + SET(TD_VER_GIT "no git commit id") ENDIF () IF (DEFINED GITINFOI) SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSEIF (HAVE_GIT) + execute_process(COMMAND git log -1 --format=%H WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMITID) + message(STATUS "git log result:${GIT_COMMITID}") + IF (GIT_COMMITID) + string (REGEX REPLACE "[\n\t\r]" "" GIT_COMMITID ${GIT_COMMITID}) + SET(TD_VER_GIT_INTERNAL ${GIT_COMMITID}) + ELSE () + message(STATUS "not a git repository") + SET(TD_VER_GIT "no git commit id") + ENDIF () ELSE () - SET(TD_VER_GIT_INTERNAL "internal") + message(STATUS "no git cmd") + SET(TD_VER_GIT_INTERNAL "no git commit id") ENDIF () IF (DEFINED VERDATE) diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt index 1d725add214f0e96d1a60aa9b889899790a4aef1..cfc17442f5c21c2d002ba42c45ce523c80eb957f 100644 --- a/deps/CMakeLists.txt +++ b/deps/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(zlib-1.2.11) @@ -9,7 +9,12 @@ ADD_SUBDIRECTORY(lz4) ADD_SUBDIRECTORY(cJson) ADD_SUBDIRECTORY(wepoll) ADD_SUBDIRECTORY(MsvcLibX) +ADD_SUBDIRECTORY(rmonotonic) IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(MQTT-C) -ENDIF () \ No newline at end of file +ENDIF () + +IF (TD_DARWIN AND TD_MQTT) + ADD_SUBDIRECTORY(MQTT-C) +ENDIF () diff --git a/deps/MQTT-C/CMakeLists.txt b/deps/MQTT-C/CMakeLists.txt index 36ede467acfa05f893cbe99ed518458d57986c79..15b35525210ec90e6e2efbdcd0e6128cb4d34f91 100644 --- a/deps/MQTT-C/CMakeLists.txt +++ b/deps/MQTT-C/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) # MQTT-C build options option(MQTT_C_OpenSSL_SUPPORT "Build MQTT-C with OpenSSL support?" OFF) diff --git a/deps/MsvcLibX/CMakeLists.txt b/deps/MsvcLibX/CMakeLists.txt index c02e4c7a4d89cdf657756ec9786d5a624419d182..4428579e1c098425c9d72d7d58a5fda15cd34c93 100644 --- a/deps/MsvcLibX/CMakeLists.txt +++ b/deps/MsvcLibX/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/MsvcLibX/include/msvcTime.h b/deps/MsvcLibX/include/msvcTime.h index 1897da58570cf503b05502d0a15c899f1f126015..6f8b5dac8f88e4c3dd40445f1c5512ba2e6e796f 100644 --- a/deps/MsvcLibX/include/msvcTime.h +++ b/deps/MsvcLibX/include/msvcTime.h @@ -38,6 +38,7 @@ typedef int clockid_t; /* Supported values for clockid_t */ #define CLOCK_REALTIME 0 +#define CLOCK_MONOTONIC 1 int clock_gettime(clockid_t clock_id, struct timespec *tp); diff --git a/deps/MsvcLibX/include/msvcUnistd.h b/deps/MsvcLibX/include/msvcUnistd.h index 9ad60625e07be8e9b749d1951375a3a501832366..9b59bae7f026d1daff65641f665f1df4ca6eb3b5 100644 --- a/deps/MsvcLibX/include/msvcUnistd.h +++ b/deps/MsvcLibX/include/msvcUnistd.h @@ -89,11 +89,12 @@ pid_t getppid(void); /* Get parent PID */ /* Path management */ #if defined(_WIN32) -#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) #define realpath realpathU +#if defined(_UTF8_SOURCE) || defined(_BSD_SOURCE) || defined(_GNU_SOURCE) +// #define realpath realpathU #define CompactPath CompactPathU #else /* _ANSI_SOURCE */ -#define realpath realpathA +// #define realpath realpathA #define CompactPath CompactPathA #endif #endif /* defined(_WIN32) */ diff --git a/deps/MsvcLibX/src/clock_gettime.c b/deps/MsvcLibX/src/clock_gettime.c index fee7532e2e552ec08b615e61e3f4d79276ea6cfc..cbddf7107ae3733186c014c26a769f3f9d386d0e 100644 --- a/deps/MsvcLibX/src/clock_gettime.c +++ b/deps/MsvcLibX/src/clock_gettime.c @@ -34,15 +34,56 @@ #include "msvcTime.h" #include "sys/msvcStat.h" /* For MsvcLibX's Filetime2Timespec */ -int clock_gettime(clockid_t clock_id, struct timespec *pTS) { - FILETIME ft; - if (clock_id != CLOCK_REALTIME) { - errno = EINVAL; - return -1; +#define MS_PER_SEC 1000ULL // MS = milliseconds +#define US_PER_MS 1000ULL // US = microseconds +#define HNS_PER_US 10ULL // HNS = hundred-nanoseconds (e.g., 1 hns = 100 ns) +#define NS_PER_US 1000ULL + +#define HNS_PER_SEC (MS_PER_SEC * US_PER_MS * HNS_PER_US) +#define NS_PER_HNS (100ULL) // NS = nanoseconds +#define NS_PER_SEC (MS_PER_SEC * US_PER_MS * NS_PER_US) + +int clock_gettime_monotonic(struct timespec *tv) { + static LARGE_INTEGER ticksPerSec; + LARGE_INTEGER ticks; + double seconds; + + if (!ticksPerSec.QuadPart) { + QueryPerformanceFrequency(&ticksPerSec); + if (!ticksPerSec.QuadPart) { + errno = ENOTSUP; + return -1; + } } + + QueryPerformanceCounter(&ticks); + + seconds = (double) ticks.QuadPart / (double) ticksPerSec.QuadPart; + tv->tv_sec = (time_t)seconds; + tv->tv_nsec = (long)((ULONGLONG)(seconds * NS_PER_SEC) % NS_PER_SEC); + + return 0; +} + +int clock_gettime_realtime(struct timespec *pTS) { + FILETIME ft; + GetSystemTimeAsFileTime(&ft); Filetime2Timespec(&ft, pTS); + return 0; } +int clock_gettime(clockid_t clock_id, struct timespec *pTS) { + if (clock_id == CLOCK_MONOTONIC) { + return clock_gettime_monotonic(pTS); + } else if (clock_id == CLOCK_REALTIME) { + return clock_gettime_realtime(pTS); + } + + errno = ENOTSUP; + + return -1; +} + #endif /* defined(_WIN32) */ diff --git a/deps/iconv/CMakeLists.txt b/deps/iconv/CMakeLists.txt index f26f109735ee81bc999b9539185ce60ccb976bfd..286070fa9071f8fcd1949850cec87c1ced3245d7 100644 --- a/deps/iconv/CMakeLists.txt +++ b/deps/iconv/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/libcurl/include/curl/curl.h b/deps/libcurl/include/curl/curl.h deleted file mode 100644 index 84229bb698b77a5ed5d67057772ed23188e191e8..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/curl.h +++ /dev/null @@ -1,2415 +0,0 @@ -#ifndef __CURL_CURL_H -#define __CURL_CURL_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ - -/* - * If you have libcurl problems, all docs and details are found here: - * http://curl.haxx.se/libcurl/ - * - * curl-library mailing list subscription and unsubscription web interface: - * http://cool.haxx.se/mailman/listinfo/curl-library/ - */ - -#include "curlver.h" /* libcurl version defines */ -#include "curlbuild.h" /* libcurl build definitions */ -#include "curlrules.h" /* libcurl rules enforcement */ - -/* - * Define WIN32 when build target is Win32 API - */ - -#if (defined(_WIN32) || defined(__WIN32__)) && \ - !defined(WIN32) && !defined(__SYMBIAN32__) -#define WIN32 -#endif - -#include -#include - -#if defined(__FreeBSD__) && (__FreeBSD__ >= 2) -/* Needed for __FreeBSD_version symbol definition */ -#include -#endif - -/* The include stuff here below is mainly for time_t! */ -#include -#include - -#if defined(WIN32) && !defined(_WIN32_WCE) && !defined(__CYGWIN__) -#if !(defined(_WINSOCKAPI_) || defined(_WINSOCK_H) || \ - defined(__LWIP_OPT_H__) || defined(LWIP_HDR_OPT_H)) -/* The check above prevents the winsock2 inclusion if winsock.h already was - included, since they can't co-exist without problems */ -#include -#include -#endif -#endif - -/* HP-UX systems version 9, 10 and 11 lack sys/select.h and so does oldish - libc5-based Linux systems. Only include it on systems that are known to - require it! */ -#if defined(_AIX) || defined(__NOVELL_LIBC__) || defined(__NetBSD__) || \ - defined(__minix) || defined(__SYMBIAN32__) || defined(__INTEGRITY) || \ - defined(ANDROID) || defined(__ANDROID__) || defined(__OpenBSD__) || \ - (defined(__FreeBSD_version) && (__FreeBSD_version < 800000)) -#include -#endif - -#if !defined(WIN32) && !defined(_WIN32_WCE) -#include -#endif - -#if !defined(WIN32) && !defined(__WATCOMC__) && !defined(__VXWORKS__) -#include -#endif - -#ifdef __BEOS__ -#include -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void CURL; - -/* - * libcurl external API function linkage decorations. - */ - -#ifdef CURL_STATICLIB -# define CURL_EXTERN -#elif defined(WIN32) || defined(_WIN32) || defined(__SYMBIAN32__) -# if defined(BUILDING_LIBCURL) -# define CURL_EXTERN __declspec(dllexport) -# else -# define CURL_EXTERN __declspec(dllimport) -# endif -#elif defined(BUILDING_LIBCURL) && defined(CURL_HIDDEN_SYMBOLS) -# define CURL_EXTERN CURL_EXTERN_SYMBOL -#else -# define CURL_EXTERN -#endif - -#ifndef curl_socket_typedef -/* socket typedef */ -#if defined(WIN32) && !defined(__LWIP_OPT_H__) && !defined(LWIP_HDR_OPT_H) -typedef SOCKET curl_socket_t; -#define CURL_SOCKET_BAD INVALID_SOCKET -#else -typedef int curl_socket_t; -#define CURL_SOCKET_BAD -1 -#endif -#define curl_socket_typedef -#endif /* curl_socket_typedef */ - -struct curl_httppost { - struct curl_httppost *next; /* next entry in the list */ - char *name; /* pointer to allocated name */ - long namelength; /* length of name length */ - char *contents; /* pointer to allocated data contents */ - long contentslength; /* length of contents field, see also - CURL_HTTPPOST_LARGE */ - char *buffer; /* pointer to allocated buffer contents */ - long bufferlength; /* length of buffer field */ - char *contenttype; /* Content-Type */ - struct curl_slist* contentheader; /* list of extra headers for this form */ - struct curl_httppost *more; /* if one field name has more than one - file, this link should link to following - files */ - long flags; /* as defined below */ - -/* specified content is a file name */ -#define CURL_HTTPPOST_FILENAME (1<<0) -/* specified content is a file name */ -#define CURL_HTTPPOST_READFILE (1<<1) -/* name is only stored pointer do not free in formfree */ -#define CURL_HTTPPOST_PTRNAME (1<<2) -/* contents is only stored pointer do not free in formfree */ -#define CURL_HTTPPOST_PTRCONTENTS (1<<3) -/* upload file from buffer */ -#define CURL_HTTPPOST_BUFFER (1<<4) -/* upload file from pointer contents */ -#define CURL_HTTPPOST_PTRBUFFER (1<<5) -/* upload file contents by using the regular read callback to get the data and - pass the given pointer as custom pointer */ -#define CURL_HTTPPOST_CALLBACK (1<<6) -/* use size in 'contentlen', added in 7.46.0 */ -#define CURL_HTTPPOST_LARGE (1<<7) - - char *showfilename; /* The file name to show. If not set, the - actual file name will be used (if this - is a file part) */ - void *userp; /* custom pointer used for - HTTPPOST_CALLBACK posts */ - curl_off_t contentlen; /* alternative length of contents - field. Used if CURL_HTTPPOST_LARGE is - set. Added in 7.46.0 */ -}; - -/* This is the CURLOPT_PROGRESSFUNCTION callback proto. It is now considered - deprecated but was the only choice up until 7.31.0 */ -typedef int (*curl_progress_callback)(void *clientp, - double dltotal, - double dlnow, - double ultotal, - double ulnow); - -/* This is the CURLOPT_XFERINFOFUNCTION callback proto. It was introduced in - 7.32.0, it avoids floating point and provides more detailed information. */ -typedef int (*curl_xferinfo_callback)(void *clientp, - curl_off_t dltotal, - curl_off_t dlnow, - curl_off_t ultotal, - curl_off_t ulnow); - -#ifndef CURL_MAX_WRITE_SIZE - /* Tests have proven that 20K is a very bad buffer size for uploads on - Windows, while 16K for some odd reason performed a lot better. - We do the ifndef check to allow this value to easier be changed at build - time for those who feel adventurous. The practical minimum is about - 400 bytes since libcurl uses a buffer of this size as a scratch area - (unrelated to network send operations). */ -#define CURL_MAX_WRITE_SIZE 16384 -#endif - -#ifndef CURL_MAX_HTTP_HEADER -/* The only reason to have a max limit for this is to avoid the risk of a bad - server feeding libcurl with a never-ending header that will cause reallocs - infinitely */ -#define CURL_MAX_HTTP_HEADER (100*1024) -#endif - -/* This is a magic return code for the write callback that, when returned, - will signal libcurl to pause receiving on the current transfer. */ -#define CURL_WRITEFUNC_PAUSE 0x10000001 - -typedef size_t (*curl_write_callback)(char *buffer, - size_t size, - size_t nitems, - void *outstream); - - - -/* enumeration of file types */ -typedef enum { - CURLFILETYPE_FILE = 0, - CURLFILETYPE_DIRECTORY, - CURLFILETYPE_SYMLINK, - CURLFILETYPE_DEVICE_BLOCK, - CURLFILETYPE_DEVICE_CHAR, - CURLFILETYPE_NAMEDPIPE, - CURLFILETYPE_SOCKET, - CURLFILETYPE_DOOR, /* is possible only on Sun Solaris now */ - - CURLFILETYPE_UNKNOWN /* should never occur */ -} curlfiletype; - -#define CURLFINFOFLAG_KNOWN_FILENAME (1<<0) -#define CURLFINFOFLAG_KNOWN_FILETYPE (1<<1) -#define CURLFINFOFLAG_KNOWN_TIME (1<<2) -#define CURLFINFOFLAG_KNOWN_PERM (1<<3) -#define CURLFINFOFLAG_KNOWN_UID (1<<4) -#define CURLFINFOFLAG_KNOWN_GID (1<<5) -#define CURLFINFOFLAG_KNOWN_SIZE (1<<6) -#define CURLFINFOFLAG_KNOWN_HLINKCOUNT (1<<7) - -/* Content of this structure depends on information which is known and is - achievable (e.g. by FTP LIST parsing). Please see the url_easy_setopt(3) man - page for callbacks returning this structure -- some fields are mandatory, - some others are optional. The FLAG field has special meaning. */ -struct curl_fileinfo { - char *filename; - curlfiletype filetype; - time_t time; - unsigned int perm; - int uid; - int gid; - curl_off_t size; - long int hardlinks; - - struct { - /* If some of these fields is not NULL, it is a pointer to b_data. */ - char *time; - char *perm; - char *user; - char *group; - char *target; /* pointer to the target filename of a symlink */ - } strings; - - unsigned int flags; - - /* used internally */ - char * b_data; - size_t b_size; - size_t b_used; -}; - -/* return codes for CURLOPT_CHUNK_BGN_FUNCTION */ -#define CURL_CHUNK_BGN_FUNC_OK 0 -#define CURL_CHUNK_BGN_FUNC_FAIL 1 /* tell the lib to end the task */ -#define CURL_CHUNK_BGN_FUNC_SKIP 2 /* skip this chunk over */ - -/* if splitting of data transfer is enabled, this callback is called before - download of an individual chunk started. Note that parameter "remains" works - only for FTP wildcard downloading (for now), otherwise is not used */ -typedef long (*curl_chunk_bgn_callback)(const void *transfer_info, - void *ptr, - int remains); - -/* return codes for CURLOPT_CHUNK_END_FUNCTION */ -#define CURL_CHUNK_END_FUNC_OK 0 -#define CURL_CHUNK_END_FUNC_FAIL 1 /* tell the lib to end the task */ - -/* If splitting of data transfer is enabled this callback is called after - download of an individual chunk finished. - Note! After this callback was set then it have to be called FOR ALL chunks. - Even if downloading of this chunk was skipped in CHUNK_BGN_FUNC. - This is the reason why we don't need "transfer_info" parameter in this - callback and we are not interested in "remains" parameter too. */ -typedef long (*curl_chunk_end_callback)(void *ptr); - -/* return codes for FNMATCHFUNCTION */ -#define CURL_FNMATCHFUNC_MATCH 0 /* string corresponds to the pattern */ -#define CURL_FNMATCHFUNC_NOMATCH 1 /* pattern doesn't match the string */ -#define CURL_FNMATCHFUNC_FAIL 2 /* an error occurred */ - -/* callback type for wildcard downloading pattern matching. If the - string matches the pattern, return CURL_FNMATCHFUNC_MATCH value, etc. */ -typedef int (*curl_fnmatch_callback)(void *ptr, - const char *pattern, - const char *string); - -/* These are the return codes for the seek callbacks */ -#define CURL_SEEKFUNC_OK 0 -#define CURL_SEEKFUNC_FAIL 1 /* fail the entire transfer */ -#define CURL_SEEKFUNC_CANTSEEK 2 /* tell libcurl seeking can't be done, so - libcurl might try other means instead */ -typedef int (*curl_seek_callback)(void *instream, - curl_off_t offset, - int origin); /* 'whence' */ - -/* This is a return code for the read callback that, when returned, will - signal libcurl to immediately abort the current transfer. */ -#define CURL_READFUNC_ABORT 0x10000000 -/* This is a return code for the read callback that, when returned, will - signal libcurl to pause sending data on the current transfer. */ -#define CURL_READFUNC_PAUSE 0x10000001 - -typedef size_t (*curl_read_callback)(char *buffer, - size_t size, - size_t nitems, - void *instream); - -typedef enum { - CURLSOCKTYPE_IPCXN, /* socket created for a specific IP connection */ - CURLSOCKTYPE_ACCEPT, /* socket created by accept() call */ - CURLSOCKTYPE_LAST /* never use */ -} curlsocktype; - -/* The return code from the sockopt_callback can signal information back - to libcurl: */ -#define CURL_SOCKOPT_OK 0 -#define CURL_SOCKOPT_ERROR 1 /* causes libcurl to abort and return - CURLE_ABORTED_BY_CALLBACK */ -#define CURL_SOCKOPT_ALREADY_CONNECTED 2 - -typedef int (*curl_sockopt_callback)(void *clientp, - curl_socket_t curlfd, - curlsocktype purpose); - -struct curl_sockaddr { - int family; - int socktype; - int protocol; - unsigned int addrlen; /* addrlen was a socklen_t type before 7.18.0 but it - turned really ugly and painful on the systems that - lack this type */ - struct sockaddr addr; -}; - -typedef curl_socket_t -(*curl_opensocket_callback)(void *clientp, - curlsocktype purpose, - struct curl_sockaddr *address); - -typedef int -(*curl_closesocket_callback)(void *clientp, curl_socket_t item); - -typedef enum { - CURLIOE_OK, /* I/O operation successful */ - CURLIOE_UNKNOWNCMD, /* command was unknown to callback */ - CURLIOE_FAILRESTART, /* failed to restart the read */ - CURLIOE_LAST /* never use */ -} curlioerr; - -typedef enum { - CURLIOCMD_NOP, /* no operation */ - CURLIOCMD_RESTARTREAD, /* restart the read stream from start */ - CURLIOCMD_LAST /* never use */ -} curliocmd; - -typedef curlioerr (*curl_ioctl_callback)(CURL *handle, - int cmd, - void *clientp); - -/* - * The following typedef's are signatures of malloc, free, realloc, strdup and - * calloc respectively. Function pointers of these types can be passed to the - * curl_global_init_mem() function to set user defined memory management - * callback routines. - */ -typedef void *(*curl_malloc_callback)(size_t size); -typedef void (*curl_free_callback)(void *ptr); -typedef void *(*curl_realloc_callback)(void *ptr, size_t size); -typedef char *(*curl_strdup_callback)(const char *str); -typedef void *(*curl_calloc_callback)(size_t nmemb, size_t size); - -/* the kind of data that is passed to information_callback*/ -typedef enum { - CURLINFO_TEXT = 0, - CURLINFO_HEADER_IN, /* 1 */ - CURLINFO_HEADER_OUT, /* 2 */ - CURLINFO_DATA_IN, /* 3 */ - CURLINFO_DATA_OUT, /* 4 */ - CURLINFO_SSL_DATA_IN, /* 5 */ - CURLINFO_SSL_DATA_OUT, /* 6 */ - CURLINFO_END -} curl_infotype; - -typedef int (*curl_debug_callback) - (CURL *handle, /* the handle/transfer this concerns */ - curl_infotype type, /* what kind of data */ - char *data, /* points to the data */ - size_t size, /* size of the data pointed to */ - void *userptr); /* whatever the user please */ - -/* All possible error codes from all sorts of curl functions. Future versions - may return other values, stay prepared. - - Always add new return codes last. Never *EVER* remove any. The return - codes must remain the same! - */ - -typedef enum { - CURLE_OK = 0, - CURLE_UNSUPPORTED_PROTOCOL, /* 1 */ - CURLE_FAILED_INIT, /* 2 */ - CURLE_URL_MALFORMAT, /* 3 */ - CURLE_NOT_BUILT_IN, /* 4 - [was obsoleted in August 2007 for - 7.17.0, reused in April 2011 for 7.21.5] */ - CURLE_COULDNT_RESOLVE_PROXY, /* 5 */ - CURLE_COULDNT_RESOLVE_HOST, /* 6 */ - CURLE_COULDNT_CONNECT, /* 7 */ - CURLE_FTP_WEIRD_SERVER_REPLY, /* 8 */ - CURLE_REMOTE_ACCESS_DENIED, /* 9 a service was denied by the server - due to lack of access - when login fails - this is not returned. */ - CURLE_FTP_ACCEPT_FAILED, /* 10 - [was obsoleted in April 2006 for - 7.15.4, reused in Dec 2011 for 7.24.0]*/ - CURLE_FTP_WEIRD_PASS_REPLY, /* 11 */ - CURLE_FTP_ACCEPT_TIMEOUT, /* 12 - timeout occurred accepting server - [was obsoleted in August 2007 for 7.17.0, - reused in Dec 2011 for 7.24.0]*/ - CURLE_FTP_WEIRD_PASV_REPLY, /* 13 */ - CURLE_FTP_WEIRD_227_FORMAT, /* 14 */ - CURLE_FTP_CANT_GET_HOST, /* 15 */ - CURLE_HTTP2, /* 16 - A problem in the http2 framing layer. - [was obsoleted in August 2007 for 7.17.0, - reused in July 2014 for 7.38.0] */ - CURLE_FTP_COULDNT_SET_TYPE, /* 17 */ - CURLE_PARTIAL_FILE, /* 18 */ - CURLE_FTP_COULDNT_RETR_FILE, /* 19 */ - CURLE_OBSOLETE20, /* 20 - NOT USED */ - CURLE_QUOTE_ERROR, /* 21 - quote command failure */ - CURLE_HTTP_RETURNED_ERROR, /* 22 */ - CURLE_WRITE_ERROR, /* 23 */ - CURLE_OBSOLETE24, /* 24 - NOT USED */ - CURLE_UPLOAD_FAILED, /* 25 - failed upload "command" */ - CURLE_READ_ERROR, /* 26 - couldn't open/read from file */ - CURLE_OUT_OF_MEMORY, /* 27 */ - /* Note: CURLE_OUT_OF_MEMORY may sometimes indicate a conversion error - instead of a memory allocation error if CURL_DOES_CONVERSIONS - is defined - */ - CURLE_OPERATION_TIMEDOUT, /* 28 - the timeout time was reached */ - CURLE_OBSOLETE29, /* 29 - NOT USED */ - CURLE_FTP_PORT_FAILED, /* 30 - FTP PORT operation failed */ - CURLE_FTP_COULDNT_USE_REST, /* 31 - the REST command failed */ - CURLE_OBSOLETE32, /* 32 - NOT USED */ - CURLE_RANGE_ERROR, /* 33 - RANGE "command" didn't work */ - CURLE_HTTP_POST_ERROR, /* 34 */ - CURLE_SSL_CONNECT_ERROR, /* 35 - wrong when connecting with SSL */ - CURLE_BAD_DOWNLOAD_RESUME, /* 36 - couldn't resume download */ - CURLE_FILE_COULDNT_READ_FILE, /* 37 */ - CURLE_LDAP_CANNOT_BIND, /* 38 */ - CURLE_LDAP_SEARCH_FAILED, /* 39 */ - CURLE_OBSOLETE40, /* 40 - NOT USED */ - CURLE_FUNCTION_NOT_FOUND, /* 41 */ - CURLE_ABORTED_BY_CALLBACK, /* 42 */ - CURLE_BAD_FUNCTION_ARGUMENT, /* 43 */ - CURLE_OBSOLETE44, /* 44 - NOT USED */ - CURLE_INTERFACE_FAILED, /* 45 - CURLOPT_INTERFACE failed */ - CURLE_OBSOLETE46, /* 46 - NOT USED */ - CURLE_TOO_MANY_REDIRECTS , /* 47 - catch endless re-direct loops */ - CURLE_UNKNOWN_OPTION, /* 48 - User specified an unknown option */ - CURLE_TELNET_OPTION_SYNTAX , /* 49 - Malformed telnet option */ - CURLE_OBSOLETE50, /* 50 - NOT USED */ - CURLE_PEER_FAILED_VERIFICATION, /* 51 - peer's certificate or fingerprint - wasn't verified fine */ - CURLE_GOT_NOTHING, /* 52 - when this is a specific error */ - CURLE_SSL_ENGINE_NOTFOUND, /* 53 - SSL crypto engine not found */ - CURLE_SSL_ENGINE_SETFAILED, /* 54 - can not set SSL crypto engine as - default */ - CURLE_SEND_ERROR, /* 55 - failed sending network data */ - CURLE_RECV_ERROR, /* 56 - failure in receiving network data */ - CURLE_OBSOLETE57, /* 57 - NOT IN USE */ - CURLE_SSL_CERTPROBLEM, /* 58 - problem with the local certificate */ - CURLE_SSL_CIPHER, /* 59 - couldn't use specified cipher */ - CURLE_SSL_CACERT, /* 60 - problem with the CA cert (path?) */ - CURLE_BAD_CONTENT_ENCODING, /* 61 - Unrecognized/bad encoding */ - CURLE_LDAP_INVALID_URL, /* 62 - Invalid LDAP URL */ - CURLE_FILESIZE_EXCEEDED, /* 63 - Maximum file size exceeded */ - CURLE_USE_SSL_FAILED, /* 64 - Requested FTP SSL level failed */ - CURLE_SEND_FAIL_REWIND, /* 65 - Sending the data requires a rewind - that failed */ - CURLE_SSL_ENGINE_INITFAILED, /* 66 - failed to initialise ENGINE */ - CURLE_LOGIN_DENIED, /* 67 - user, password or similar was not - accepted and we failed to login */ - CURLE_TFTP_NOTFOUND, /* 68 - file not found on server */ - CURLE_TFTP_PERM, /* 69 - permission problem on server */ - CURLE_REMOTE_DISK_FULL, /* 70 - out of disk space on server */ - CURLE_TFTP_ILLEGAL, /* 71 - Illegal TFTP operation */ - CURLE_TFTP_UNKNOWNID, /* 72 - Unknown transfer ID */ - CURLE_REMOTE_FILE_EXISTS, /* 73 - File already exists */ - CURLE_TFTP_NOSUCHUSER, /* 74 - No such user */ - CURLE_CONV_FAILED, /* 75 - conversion failed */ - CURLE_CONV_REQD, /* 76 - caller must register conversion - callbacks using curl_easy_setopt options - CURLOPT_CONV_FROM_NETWORK_FUNCTION, - CURLOPT_CONV_TO_NETWORK_FUNCTION, and - CURLOPT_CONV_FROM_UTF8_FUNCTION */ - CURLE_SSL_CACERT_BADFILE, /* 77 - could not load CACERT file, missing - or wrong format */ - CURLE_REMOTE_FILE_NOT_FOUND, /* 78 - remote file not found */ - CURLE_SSH, /* 79 - error from the SSH layer, somewhat - generic so the error message will be of - interest when this has happened */ - - CURLE_SSL_SHUTDOWN_FAILED, /* 80 - Failed to shut down the SSL - connection */ - CURLE_AGAIN, /* 81 - socket is not ready for send/recv, - wait till it's ready and try again (Added - in 7.18.2) */ - CURLE_SSL_CRL_BADFILE, /* 82 - could not load CRL file, missing or - wrong format (Added in 7.19.0) */ - CURLE_SSL_ISSUER_ERROR, /* 83 - Issuer check failed. (Added in - 7.19.0) */ - CURLE_FTP_PRET_FAILED, /* 84 - a PRET command failed */ - CURLE_RTSP_CSEQ_ERROR, /* 85 - mismatch of RTSP CSeq numbers */ - CURLE_RTSP_SESSION_ERROR, /* 86 - mismatch of RTSP Session Ids */ - CURLE_FTP_BAD_FILE_LIST, /* 87 - unable to parse FTP file list */ - CURLE_CHUNK_FAILED, /* 88 - chunk callback reported error */ - CURLE_NO_CONNECTION_AVAILABLE, /* 89 - No connection available, the - session will be queued */ - CURLE_SSL_PINNEDPUBKEYNOTMATCH, /* 90 - specified pinned public key did not - match */ - CURLE_SSL_INVALIDCERTSTATUS, /* 91 - invalid certificate status */ - CURL_LAST /* never use! */ -} CURLcode; - -#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all - the obsolete stuff removed! */ - -/* Previously obsolete error code re-used in 7.38.0 */ -#define CURLE_OBSOLETE16 CURLE_HTTP2 - -/* Previously obsolete error codes re-used in 7.24.0 */ -#define CURLE_OBSOLETE10 CURLE_FTP_ACCEPT_FAILED -#define CURLE_OBSOLETE12 CURLE_FTP_ACCEPT_TIMEOUT - -/* compatibility with older names */ -#define CURLOPT_ENCODING CURLOPT_ACCEPT_ENCODING - -/* The following were added in 7.21.5, April 2011 */ -#define CURLE_UNKNOWN_TELNET_OPTION CURLE_UNKNOWN_OPTION - -/* The following were added in 7.17.1 */ -/* These are scheduled to disappear by 2009 */ -#define CURLE_SSL_PEER_CERTIFICATE CURLE_PEER_FAILED_VERIFICATION - -/* The following were added in 7.17.0 */ -/* These are scheduled to disappear by 2009 */ -#define CURLE_OBSOLETE CURLE_OBSOLETE50 /* no one should be using this! */ -#define CURLE_BAD_PASSWORD_ENTERED CURLE_OBSOLETE46 -#define CURLE_BAD_CALLING_ORDER CURLE_OBSOLETE44 -#define CURLE_FTP_USER_PASSWORD_INCORRECT CURLE_OBSOLETE10 -#define CURLE_FTP_CANT_RECONNECT CURLE_OBSOLETE16 -#define CURLE_FTP_COULDNT_GET_SIZE CURLE_OBSOLETE32 -#define CURLE_FTP_COULDNT_SET_ASCII CURLE_OBSOLETE29 -#define CURLE_FTP_WEIRD_USER_REPLY CURLE_OBSOLETE12 -#define CURLE_FTP_WRITE_ERROR CURLE_OBSOLETE20 -#define CURLE_LIBRARY_NOT_FOUND CURLE_OBSOLETE40 -#define CURLE_MALFORMAT_USER CURLE_OBSOLETE24 -#define CURLE_SHARE_IN_USE CURLE_OBSOLETE57 -#define CURLE_URL_MALFORMAT_USER CURLE_NOT_BUILT_IN - -#define CURLE_FTP_ACCESS_DENIED CURLE_REMOTE_ACCESS_DENIED -#define CURLE_FTP_COULDNT_SET_BINARY CURLE_FTP_COULDNT_SET_TYPE -#define CURLE_FTP_QUOTE_ERROR CURLE_QUOTE_ERROR -#define CURLE_TFTP_DISKFULL CURLE_REMOTE_DISK_FULL -#define CURLE_TFTP_EXISTS CURLE_REMOTE_FILE_EXISTS -#define CURLE_HTTP_RANGE_ERROR CURLE_RANGE_ERROR -#define CURLE_FTP_SSL_FAILED CURLE_USE_SSL_FAILED - -/* The following were added earlier */ - -#define CURLE_OPERATION_TIMEOUTED CURLE_OPERATION_TIMEDOUT - -#define CURLE_HTTP_NOT_FOUND CURLE_HTTP_RETURNED_ERROR -#define CURLE_HTTP_PORT_FAILED CURLE_INTERFACE_FAILED -#define CURLE_FTP_COULDNT_STOR_FILE CURLE_UPLOAD_FAILED - -#define CURLE_FTP_PARTIAL_FILE CURLE_PARTIAL_FILE -#define CURLE_FTP_BAD_DOWNLOAD_RESUME CURLE_BAD_DOWNLOAD_RESUME - -/* This was the error code 50 in 7.7.3 and a few earlier versions, this - is no longer used by libcurl but is instead #defined here only to not - make programs break */ -#define CURLE_ALREADY_COMPLETE 99999 - -/* Provide defines for really old option names */ -#define CURLOPT_FILE CURLOPT_WRITEDATA /* name changed in 7.9.7 */ -#define CURLOPT_INFILE CURLOPT_READDATA /* name changed in 7.9.7 */ -#define CURLOPT_WRITEHEADER CURLOPT_HEADERDATA - -/* Since long deprecated options with no code in the lib that does anything - with them. */ -#define CURLOPT_WRITEINFO CURLOPT_OBSOLETE40 -#define CURLOPT_CLOSEPOLICY CURLOPT_OBSOLETE72 - -#endif /*!CURL_NO_OLDIES*/ - -/* This prototype applies to all conversion callbacks */ -typedef CURLcode (*curl_conv_callback)(char *buffer, size_t length); - -typedef CURLcode (*curl_ssl_ctx_callback)(CURL *curl, /* easy handle */ - void *ssl_ctx, /* actually an - OpenSSL SSL_CTX */ - void *userptr); - -typedef enum { - CURLPROXY_HTTP = 0, /* added in 7.10, new in 7.19.4 default is to use - CONNECT HTTP/1.1 */ - CURLPROXY_HTTP_1_0 = 1, /* added in 7.19.4, force to use CONNECT - HTTP/1.0 */ - CURLPROXY_SOCKS4 = 4, /* support added in 7.15.2, enum existed already - in 7.10 */ - CURLPROXY_SOCKS5 = 5, /* added in 7.10 */ - CURLPROXY_SOCKS4A = 6, /* added in 7.18.0 */ - CURLPROXY_SOCKS5_HOSTNAME = 7 /* Use the SOCKS5 protocol but pass along the - host name rather than the IP address. added - in 7.18.0 */ -} curl_proxytype; /* this enum was added in 7.10 */ - -/* - * Bitmasks for CURLOPT_HTTPAUTH and CURLOPT_PROXYAUTH options: - * - * CURLAUTH_NONE - No HTTP authentication - * CURLAUTH_BASIC - HTTP Basic authentication (default) - * CURLAUTH_DIGEST - HTTP Digest authentication - * CURLAUTH_NEGOTIATE - HTTP Negotiate (SPNEGO) authentication - * CURLAUTH_GSSNEGOTIATE - Alias for CURLAUTH_NEGOTIATE (deprecated) - * CURLAUTH_NTLM - HTTP NTLM authentication - * CURLAUTH_DIGEST_IE - HTTP Digest authentication with IE flavour - * CURLAUTH_NTLM_WB - HTTP NTLM authentication delegated to winbind helper - * CURLAUTH_ONLY - Use together with a single other type to force no - * authentication or just that single type - * CURLAUTH_ANY - All fine types set - * CURLAUTH_ANYSAFE - All fine types except Basic - */ - -#define CURLAUTH_NONE ((unsigned long)0) -#define CURLAUTH_BASIC (((unsigned long)1)<<0) -#define CURLAUTH_DIGEST (((unsigned long)1)<<1) -#define CURLAUTH_NEGOTIATE (((unsigned long)1)<<2) -/* Deprecated since the advent of CURLAUTH_NEGOTIATE */ -#define CURLAUTH_GSSNEGOTIATE CURLAUTH_NEGOTIATE -#define CURLAUTH_NTLM (((unsigned long)1)<<3) -#define CURLAUTH_DIGEST_IE (((unsigned long)1)<<4) -#define CURLAUTH_NTLM_WB (((unsigned long)1)<<5) -#define CURLAUTH_ONLY (((unsigned long)1)<<31) -#define CURLAUTH_ANY (~CURLAUTH_DIGEST_IE) -#define CURLAUTH_ANYSAFE (~(CURLAUTH_BASIC|CURLAUTH_DIGEST_IE)) - -#define CURLSSH_AUTH_ANY ~0 /* all types supported by the server */ -#define CURLSSH_AUTH_NONE 0 /* none allowed, silly but complete */ -#define CURLSSH_AUTH_PUBLICKEY (1<<0) /* public/private key files */ -#define CURLSSH_AUTH_PASSWORD (1<<1) /* password */ -#define CURLSSH_AUTH_HOST (1<<2) /* host key files */ -#define CURLSSH_AUTH_KEYBOARD (1<<3) /* keyboard interactive */ -#define CURLSSH_AUTH_AGENT (1<<4) /* agent (ssh-agent, pageant...) */ -#define CURLSSH_AUTH_DEFAULT CURLSSH_AUTH_ANY - -#define CURLGSSAPI_DELEGATION_NONE 0 /* no delegation (default) */ -#define CURLGSSAPI_DELEGATION_POLICY_FLAG (1<<0) /* if permitted by policy */ -#define CURLGSSAPI_DELEGATION_FLAG (1<<1) /* delegate always */ - -#define CURL_ERROR_SIZE 256 - -enum curl_khtype { - CURLKHTYPE_UNKNOWN, - CURLKHTYPE_RSA1, - CURLKHTYPE_RSA, - CURLKHTYPE_DSS -}; - -struct curl_khkey { - const char *key; /* points to a zero-terminated string encoded with base64 - if len is zero, otherwise to the "raw" data */ - size_t len; - enum curl_khtype keytype; -}; - -/* this is the set of return values expected from the curl_sshkeycallback - callback */ -enum curl_khstat { - CURLKHSTAT_FINE_ADD_TO_FILE, - CURLKHSTAT_FINE, - CURLKHSTAT_REJECT, /* reject the connection, return an error */ - CURLKHSTAT_DEFER, /* do not accept it, but we can't answer right now so - this causes a CURLE_DEFER error but otherwise the - connection will be left intact etc */ - CURLKHSTAT_LAST /* not for use, only a marker for last-in-list */ -}; - -/* this is the set of status codes pass in to the callback */ -enum curl_khmatch { - CURLKHMATCH_OK, /* match */ - CURLKHMATCH_MISMATCH, /* host found, key mismatch! */ - CURLKHMATCH_MISSING, /* no matching host/key found */ - CURLKHMATCH_LAST /* not for use, only a marker for last-in-list */ -}; - -typedef int - (*curl_sshkeycallback) (CURL *easy, /* easy handle */ - const struct curl_khkey *knownkey, /* known */ - const struct curl_khkey *foundkey, /* found */ - enum curl_khmatch, /* libcurl's view on the keys */ - void *clientp); /* custom pointer passed from app */ - -/* parameter for the CURLOPT_USE_SSL option */ -typedef enum { - CURLUSESSL_NONE, /* do not attempt to use SSL */ - CURLUSESSL_TRY, /* try using SSL, proceed anyway otherwise */ - CURLUSESSL_CONTROL, /* SSL for the control connection or fail */ - CURLUSESSL_ALL, /* SSL for all communication or fail */ - CURLUSESSL_LAST /* not an option, never use */ -} curl_usessl; - -/* Definition of bits for the CURLOPT_SSL_OPTIONS argument: */ - -/* - ALLOW_BEAST tells libcurl to allow the BEAST SSL vulnerability in the - name of improving interoperability with older servers. Some SSL libraries - have introduced work-arounds for this flaw but those work-arounds sometimes - make the SSL communication fail. To regain functionality with those broken - servers, a user can this way allow the vulnerability back. */ -#define CURLSSLOPT_ALLOW_BEAST (1<<0) - -/* - NO_REVOKE tells libcurl to disable certificate revocation checks for those - SSL backends where such behavior is present. */ -#define CURLSSLOPT_NO_REVOKE (1<<1) - -#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all - the obsolete stuff removed! */ - -/* Backwards compatibility with older names */ -/* These are scheduled to disappear by 2009 */ - -#define CURLFTPSSL_NONE CURLUSESSL_NONE -#define CURLFTPSSL_TRY CURLUSESSL_TRY -#define CURLFTPSSL_CONTROL CURLUSESSL_CONTROL -#define CURLFTPSSL_ALL CURLUSESSL_ALL -#define CURLFTPSSL_LAST CURLUSESSL_LAST -#define curl_ftpssl curl_usessl -#endif /*!CURL_NO_OLDIES*/ - -/* parameter for the CURLOPT_FTP_SSL_CCC option */ -typedef enum { - CURLFTPSSL_CCC_NONE, /* do not send CCC */ - CURLFTPSSL_CCC_PASSIVE, /* Let the server initiate the shutdown */ - CURLFTPSSL_CCC_ACTIVE, /* Initiate the shutdown */ - CURLFTPSSL_CCC_LAST /* not an option, never use */ -} curl_ftpccc; - -/* parameter for the CURLOPT_FTPSSLAUTH option */ -typedef enum { - CURLFTPAUTH_DEFAULT, /* let libcurl decide */ - CURLFTPAUTH_SSL, /* use "AUTH SSL" */ - CURLFTPAUTH_TLS, /* use "AUTH TLS" */ - CURLFTPAUTH_LAST /* not an option, never use */ -} curl_ftpauth; - -/* parameter for the CURLOPT_FTP_CREATE_MISSING_DIRS option */ -typedef enum { - CURLFTP_CREATE_DIR_NONE, /* do NOT create missing dirs! */ - CURLFTP_CREATE_DIR, /* (FTP/SFTP) if CWD fails, try MKD and then CWD - again if MKD succeeded, for SFTP this does - similar magic */ - CURLFTP_CREATE_DIR_RETRY, /* (FTP only) if CWD fails, try MKD and then CWD - again even if MKD failed! */ - CURLFTP_CREATE_DIR_LAST /* not an option, never use */ -} curl_ftpcreatedir; - -/* parameter for the CURLOPT_FTP_FILEMETHOD option */ -typedef enum { - CURLFTPMETHOD_DEFAULT, /* let libcurl pick */ - CURLFTPMETHOD_MULTICWD, /* single CWD operation for each path part */ - CURLFTPMETHOD_NOCWD, /* no CWD at all */ - CURLFTPMETHOD_SINGLECWD, /* one CWD to full dir, then work on file */ - CURLFTPMETHOD_LAST /* not an option, never use */ -} curl_ftpmethod; - -/* bitmask defines for CURLOPT_HEADEROPT */ -#define CURLHEADER_UNIFIED 0 -#define CURLHEADER_SEPARATE (1<<0) - -/* CURLPROTO_ defines are for the CURLOPT_*PROTOCOLS options */ -#define CURLPROTO_HTTP (1<<0) -#define CURLPROTO_HTTPS (1<<1) -#define CURLPROTO_FTP (1<<2) -#define CURLPROTO_FTPS (1<<3) -#define CURLPROTO_SCP (1<<4) -#define CURLPROTO_SFTP (1<<5) -#define CURLPROTO_TELNET (1<<6) -#define CURLPROTO_LDAP (1<<7) -#define CURLPROTO_LDAPS (1<<8) -#define CURLPROTO_DICT (1<<9) -#define CURLPROTO_FILE (1<<10) -#define CURLPROTO_TFTP (1<<11) -#define CURLPROTO_IMAP (1<<12) -#define CURLPROTO_IMAPS (1<<13) -#define CURLPROTO_POP3 (1<<14) -#define CURLPROTO_POP3S (1<<15) -#define CURLPROTO_SMTP (1<<16) -#define CURLPROTO_SMTPS (1<<17) -#define CURLPROTO_RTSP (1<<18) -#define CURLPROTO_RTMP (1<<19) -#define CURLPROTO_RTMPT (1<<20) -#define CURLPROTO_RTMPE (1<<21) -#define CURLPROTO_RTMPTE (1<<22) -#define CURLPROTO_RTMPS (1<<23) -#define CURLPROTO_RTMPTS (1<<24) -#define CURLPROTO_GOPHER (1<<25) -#define CURLPROTO_SMB (1<<26) -#define CURLPROTO_SMBS (1<<27) -#define CURLPROTO_ALL (~0) /* enable everything */ - -/* long may be 32 or 64 bits, but we should never depend on anything else - but 32 */ -#define CURLOPTTYPE_LONG 0 -#define CURLOPTTYPE_OBJECTPOINT 10000 -#define CURLOPTTYPE_STRINGPOINT 10000 -#define CURLOPTTYPE_FUNCTIONPOINT 20000 -#define CURLOPTTYPE_OFF_T 30000 - -/* *STRINGPOINT is an alias for OBJECTPOINT to allow tools to extract the - string options from the header file */ - -/* name is uppercase CURLOPT_, - type is one of the defined CURLOPTTYPE_ - number is unique identifier */ -#ifdef CINIT -#undef CINIT -#endif - -#ifdef CURL_ISOCPP -#define CINIT(na,t,nu) CURLOPT_ ## na = CURLOPTTYPE_ ## t + nu -#else -/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ -#define LONG CURLOPTTYPE_LONG -#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT -#define STRINGPOINT CURLOPTTYPE_OBJECTPOINT -#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT -#define OFF_T CURLOPTTYPE_OFF_T -#define CINIT(name,type,number) CURLOPT_/**/name = type + number -#endif - -/* - * This macro-mania below setups the CURLOPT_[what] enum, to be used with - * curl_easy_setopt(). The first argument in the CINIT() macro is the [what] - * word. - */ - -typedef enum { - /* This is the FILE * or void * the regular output should be written to. */ - CINIT(WRITEDATA, OBJECTPOINT, 1), - - /* The full URL to get/put */ - CINIT(URL, STRINGPOINT, 2), - - /* Port number to connect to, if other than default. */ - CINIT(PORT, LONG, 3), - - /* Name of proxy to use. */ - CINIT(PROXY, STRINGPOINT, 4), - - /* "user:password;options" to use when fetching. */ - CINIT(USERPWD, STRINGPOINT, 5), - - /* "user:password" to use with proxy. */ - CINIT(PROXYUSERPWD, STRINGPOINT, 6), - - /* Range to get, specified as an ASCII string. */ - CINIT(RANGE, STRINGPOINT, 7), - - /* not used */ - - /* Specified file stream to upload from (use as input): */ - CINIT(READDATA, OBJECTPOINT, 9), - - /* Buffer to receive error messages in, must be at least CURL_ERROR_SIZE - * bytes big. If this is not used, error messages go to stderr instead: */ - CINIT(ERRORBUFFER, OBJECTPOINT, 10), - - /* Function that will be called to store the output (instead of fwrite). The - * parameters will use fwrite() syntax, make sure to follow them. */ - CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11), - - /* Function that will be called to read the input (instead of fread). The - * parameters will use fread() syntax, make sure to follow them. */ - CINIT(READFUNCTION, FUNCTIONPOINT, 12), - - /* Time-out the read operation after this amount of seconds */ - CINIT(TIMEOUT, LONG, 13), - - /* If the CURLOPT_INFILE is used, this can be used to inform libcurl about - * how large the file being sent really is. That allows better error - * checking and better verifies that the upload was successful. -1 means - * unknown size. - * - * For large file support, there is also a _LARGE version of the key - * which takes an off_t type, allowing platforms with larger off_t - * sizes to handle larger files. See below for INFILESIZE_LARGE. - */ - CINIT(INFILESIZE, LONG, 14), - - /* POST static input fields. */ - CINIT(POSTFIELDS, OBJECTPOINT, 15), - - /* Set the referrer page (needed by some CGIs) */ - CINIT(REFERER, STRINGPOINT, 16), - - /* Set the FTP PORT string (interface name, named or numerical IP address) - Use i.e '-' to use default address. */ - CINIT(FTPPORT, STRINGPOINT, 17), - - /* Set the User-Agent string (examined by some CGIs) */ - CINIT(USERAGENT, STRINGPOINT, 18), - - /* If the download receives less than "low speed limit" bytes/second - * during "low speed time" seconds, the operations is aborted. - * You could i.e if you have a pretty high speed connection, abort if - * it is less than 2000 bytes/sec during 20 seconds. - */ - - /* Set the "low speed limit" */ - CINIT(LOW_SPEED_LIMIT, LONG, 19), - - /* Set the "low speed time" */ - CINIT(LOW_SPEED_TIME, LONG, 20), - - /* Set the continuation offset. - * - * Note there is also a _LARGE version of this key which uses - * off_t types, allowing for large file offsets on platforms which - * use larger-than-32-bit off_t's. Look below for RESUME_FROM_LARGE. - */ - CINIT(RESUME_FROM, LONG, 21), - - /* Set cookie in request: */ - CINIT(COOKIE, STRINGPOINT, 22), - - /* This points to a linked list of headers, struct curl_slist kind. This - list is also used for RTSP (in spite of its name) */ - CINIT(HTTPHEADER, OBJECTPOINT, 23), - - /* This points to a linked list of post entries, struct curl_httppost */ - CINIT(HTTPPOST, OBJECTPOINT, 24), - - /* name of the file keeping your private SSL-certificate */ - CINIT(SSLCERT, STRINGPOINT, 25), - - /* password for the SSL or SSH private key */ - CINIT(KEYPASSWD, STRINGPOINT, 26), - - /* send TYPE parameter? */ - CINIT(CRLF, LONG, 27), - - /* send linked-list of QUOTE commands */ - CINIT(QUOTE, OBJECTPOINT, 28), - - /* send FILE * or void * to store headers to, if you use a callback it - is simply passed to the callback unmodified */ - CINIT(HEADERDATA, OBJECTPOINT, 29), - - /* point to a file to read the initial cookies from, also enables - "cookie awareness" */ - CINIT(COOKIEFILE, STRINGPOINT, 31), - - /* What version to specifically try to use. - See CURL_SSLVERSION defines below. */ - CINIT(SSLVERSION, LONG, 32), - - /* What kind of HTTP time condition to use, see defines */ - CINIT(TIMECONDITION, LONG, 33), - - /* Time to use with the above condition. Specified in number of seconds - since 1 Jan 1970 */ - CINIT(TIMEVALUE, LONG, 34), - - /* 35 = OBSOLETE */ - - /* Custom request, for customizing the get command like - HTTP: DELETE, TRACE and others - FTP: to use a different list command - */ - CINIT(CUSTOMREQUEST, STRINGPOINT, 36), - - /* FILE handle to use instead of stderr */ - CINIT(STDERR, OBJECTPOINT, 37), - - /* 38 is not used */ - - /* send linked-list of post-transfer QUOTE commands */ - CINIT(POSTQUOTE, OBJECTPOINT, 39), - - CINIT(OBSOLETE40, OBJECTPOINT, 40), /* OBSOLETE, do not use! */ - - CINIT(VERBOSE, LONG, 41), /* talk a lot */ - CINIT(HEADER, LONG, 42), /* throw the header out too */ - CINIT(NOPROGRESS, LONG, 43), /* shut off the progress meter */ - CINIT(NOBODY, LONG, 44), /* use HEAD to get http document */ - CINIT(FAILONERROR, LONG, 45), /* no output on http error codes >= 400 */ - CINIT(UPLOAD, LONG, 46), /* this is an upload */ - CINIT(POST, LONG, 47), /* HTTP POST method */ - CINIT(DIRLISTONLY, LONG, 48), /* bare names when listing directories */ - - CINIT(APPEND, LONG, 50), /* Append instead of overwrite on upload! */ - - /* Specify whether to read the user+password from the .netrc or the URL. - * This must be one of the CURL_NETRC_* enums below. */ - CINIT(NETRC, LONG, 51), - - CINIT(FOLLOWLOCATION, LONG, 52), /* use Location: Luke! */ - - CINIT(TRANSFERTEXT, LONG, 53), /* transfer data in text/ASCII format */ - CINIT(PUT, LONG, 54), /* HTTP PUT */ - - /* 55 = OBSOLETE */ - - /* DEPRECATED - * Function that will be called instead of the internal progress display - * function. This function should be defined as the curl_progress_callback - * prototype defines. */ - CINIT(PROGRESSFUNCTION, FUNCTIONPOINT, 56), - - /* Data passed to the CURLOPT_PROGRESSFUNCTION and CURLOPT_XFERINFOFUNCTION - callbacks */ - CINIT(PROGRESSDATA, OBJECTPOINT, 57), -#define CURLOPT_XFERINFODATA CURLOPT_PROGRESSDATA - - /* We want the referrer field set automatically when following locations */ - CINIT(AUTOREFERER, LONG, 58), - - /* Port of the proxy, can be set in the proxy string as well with: - "[host]:[port]" */ - CINIT(PROXYPORT, LONG, 59), - - /* size of the POST input data, if strlen() is not good to use */ - CINIT(POSTFIELDSIZE, LONG, 60), - - /* tunnel non-http operations through a HTTP proxy */ - CINIT(HTTPPROXYTUNNEL, LONG, 61), - - /* Set the interface string to use as outgoing network interface */ - CINIT(INTERFACE, STRINGPOINT, 62), - - /* Set the krb4/5 security level, this also enables krb4/5 awareness. This - * is a string, 'clear', 'safe', 'confidential' or 'private'. If the string - * is set but doesn't match one of these, 'private' will be used. */ - CINIT(KRBLEVEL, STRINGPOINT, 63), - - /* Set if we should verify the peer in ssl handshake, set 1 to verify. */ - CINIT(SSL_VERIFYPEER, LONG, 64), - - /* The CApath or CAfile used to validate the peer certificate - this option is used only if SSL_VERIFYPEER is true */ - CINIT(CAINFO, STRINGPOINT, 65), - - /* 66 = OBSOLETE */ - /* 67 = OBSOLETE */ - - /* Maximum number of http redirects to follow */ - CINIT(MAXREDIRS, LONG, 68), - - /* Pass a long set to 1 to get the date of the requested document (if - possible)! Pass a zero to shut it off. */ - CINIT(FILETIME, LONG, 69), - - /* This points to a linked list of telnet options */ - CINIT(TELNETOPTIONS, OBJECTPOINT, 70), - - /* Max amount of cached alive connections */ - CINIT(MAXCONNECTS, LONG, 71), - - CINIT(OBSOLETE72, LONG, 72), /* OBSOLETE, do not use! */ - - /* 73 = OBSOLETE */ - - /* Set to explicitly use a new connection for the upcoming transfer. - Do not use this unless you're absolutely sure of this, as it makes the - operation slower and is less friendly for the network. */ - CINIT(FRESH_CONNECT, LONG, 74), - - /* Set to explicitly forbid the upcoming transfer's connection to be re-used - when done. Do not use this unless you're absolutely sure of this, as it - makes the operation slower and is less friendly for the network. */ - CINIT(FORBID_REUSE, LONG, 75), - - /* Set to a file name that contains random data for libcurl to use to - seed the random engine when doing SSL connects. */ - CINIT(RANDOM_FILE, STRINGPOINT, 76), - - /* Set to the Entropy Gathering Daemon socket pathname */ - CINIT(EGDSOCKET, STRINGPOINT, 77), - - /* Time-out connect operations after this amount of seconds, if connects are - OK within this time, then fine... This only aborts the connect phase. */ - CINIT(CONNECTTIMEOUT, LONG, 78), - - /* Function that will be called to store headers (instead of fwrite). The - * parameters will use fwrite() syntax, make sure to follow them. */ - CINIT(HEADERFUNCTION, FUNCTIONPOINT, 79), - - /* Set this to force the HTTP request to get back to GET. Only really usable - if POST, PUT or a custom request have been used first. - */ - CINIT(HTTPGET, LONG, 80), - - /* Set if we should verify the Common name from the peer certificate in ssl - * handshake, set 1 to check existence, 2 to ensure that it matches the - * provided hostname. */ - CINIT(SSL_VERIFYHOST, LONG, 81), - - /* Specify which file name to write all known cookies in after completed - operation. Set file name to "-" (dash) to make it go to stdout. */ - CINIT(COOKIEJAR, STRINGPOINT, 82), - - /* Specify which SSL ciphers to use */ - CINIT(SSL_CIPHER_LIST, STRINGPOINT, 83), - - /* Specify which HTTP version to use! This must be set to one of the - CURL_HTTP_VERSION* enums set below. */ - CINIT(HTTP_VERSION, LONG, 84), - - /* Specifically switch on or off the FTP engine's use of the EPSV command. By - default, that one will always be attempted before the more traditional - PASV command. */ - CINIT(FTP_USE_EPSV, LONG, 85), - - /* type of the file keeping your SSL-certificate ("DER", "PEM", "ENG") */ - CINIT(SSLCERTTYPE, STRINGPOINT, 86), - - /* name of the file keeping your private SSL-key */ - CINIT(SSLKEY, STRINGPOINT, 87), - - /* type of the file keeping your private SSL-key ("DER", "PEM", "ENG") */ - CINIT(SSLKEYTYPE, STRINGPOINT, 88), - - /* crypto engine for the SSL-sub system */ - CINIT(SSLENGINE, STRINGPOINT, 89), - - /* set the crypto engine for the SSL-sub system as default - the param has no meaning... - */ - CINIT(SSLENGINE_DEFAULT, LONG, 90), - - /* Non-zero value means to use the global dns cache */ - CINIT(DNS_USE_GLOBAL_CACHE, LONG, 91), /* DEPRECATED, do not use! */ - - /* DNS cache timeout */ - CINIT(DNS_CACHE_TIMEOUT, LONG, 92), - - /* send linked-list of pre-transfer QUOTE commands */ - CINIT(PREQUOTE, OBJECTPOINT, 93), - - /* set the debug function */ - CINIT(DEBUGFUNCTION, FUNCTIONPOINT, 94), - - /* set the data for the debug function */ - CINIT(DEBUGDATA, OBJECTPOINT, 95), - - /* mark this as start of a cookie session */ - CINIT(COOKIESESSION, LONG, 96), - - /* The CApath directory used to validate the peer certificate - this option is used only if SSL_VERIFYPEER is true */ - CINIT(CAPATH, STRINGPOINT, 97), - - /* Instruct libcurl to use a smaller receive buffer */ - CINIT(BUFFERSIZE, LONG, 98), - - /* Instruct libcurl to not use any signal/alarm handlers, even when using - timeouts. This option is useful for multi-threaded applications. - See libcurl-the-guide for more background information. */ - CINIT(NOSIGNAL, LONG, 99), - - /* Provide a CURLShare for mutexing non-ts data */ - CINIT(SHARE, OBJECTPOINT, 100), - - /* indicates type of proxy. accepted values are CURLPROXY_HTTP (default), - CURLPROXY_SOCKS4, CURLPROXY_SOCKS4A and CURLPROXY_SOCKS5. */ - CINIT(PROXYTYPE, LONG, 101), - - /* Set the Accept-Encoding string. Use this to tell a server you would like - the response to be compressed. Before 7.21.6, this was known as - CURLOPT_ENCODING */ - CINIT(ACCEPT_ENCODING, STRINGPOINT, 102), - - /* Set pointer to private data */ - CINIT(PRIVATE, OBJECTPOINT, 103), - - /* Set aliases for HTTP 200 in the HTTP Response header */ - CINIT(HTTP200ALIASES, OBJECTPOINT, 104), - - /* Continue to send authentication (user+password) when following locations, - even when hostname changed. This can potentially send off the name - and password to whatever host the server decides. */ - CINIT(UNRESTRICTED_AUTH, LONG, 105), - - /* Specifically switch on or off the FTP engine's use of the EPRT command ( - it also disables the LPRT attempt). By default, those ones will always be - attempted before the good old traditional PORT command. */ - CINIT(FTP_USE_EPRT, LONG, 106), - - /* Set this to a bitmask value to enable the particular authentications - methods you like. Use this in combination with CURLOPT_USERPWD. - Note that setting multiple bits may cause extra network round-trips. */ - CINIT(HTTPAUTH, LONG, 107), - - /* Set the ssl context callback function, currently only for OpenSSL ssl_ctx - in second argument. The function must be matching the - curl_ssl_ctx_callback proto. */ - CINIT(SSL_CTX_FUNCTION, FUNCTIONPOINT, 108), - - /* Set the userdata for the ssl context callback function's third - argument */ - CINIT(SSL_CTX_DATA, OBJECTPOINT, 109), - - /* FTP Option that causes missing dirs to be created on the remote server. - In 7.19.4 we introduced the convenience enums for this option using the - CURLFTP_CREATE_DIR prefix. - */ - CINIT(FTP_CREATE_MISSING_DIRS, LONG, 110), - - /* Set this to a bitmask value to enable the particular authentications - methods you like. Use this in combination with CURLOPT_PROXYUSERPWD. - Note that setting multiple bits may cause extra network round-trips. */ - CINIT(PROXYAUTH, LONG, 111), - - /* FTP option that changes the timeout, in seconds, associated with - getting a response. This is different from transfer timeout time and - essentially places a demand on the FTP server to acknowledge commands - in a timely manner. */ - CINIT(FTP_RESPONSE_TIMEOUT, LONG, 112), -#define CURLOPT_SERVER_RESPONSE_TIMEOUT CURLOPT_FTP_RESPONSE_TIMEOUT - - /* Set this option to one of the CURL_IPRESOLVE_* defines (see below) to - tell libcurl to resolve names to those IP versions only. This only has - affect on systems with support for more than one, i.e IPv4 _and_ IPv6. */ - CINIT(IPRESOLVE, LONG, 113), - - /* Set this option to limit the size of a file that will be downloaded from - an HTTP or FTP server. - - Note there is also _LARGE version which adds large file support for - platforms which have larger off_t sizes. See MAXFILESIZE_LARGE below. */ - CINIT(MAXFILESIZE, LONG, 114), - - /* See the comment for INFILESIZE above, but in short, specifies - * the size of the file being uploaded. -1 means unknown. - */ - CINIT(INFILESIZE_LARGE, OFF_T, 115), - - /* Sets the continuation offset. There is also a LONG version of this; - * look above for RESUME_FROM. - */ - CINIT(RESUME_FROM_LARGE, OFF_T, 116), - - /* Sets the maximum size of data that will be downloaded from - * an HTTP or FTP server. See MAXFILESIZE above for the LONG version. - */ - CINIT(MAXFILESIZE_LARGE, OFF_T, 117), - - /* Set this option to the file name of your .netrc file you want libcurl - to parse (using the CURLOPT_NETRC option). If not set, libcurl will do - a poor attempt to find the user's home directory and check for a .netrc - file in there. */ - CINIT(NETRC_FILE, STRINGPOINT, 118), - - /* Enable SSL/TLS for FTP, pick one of: - CURLUSESSL_TRY - try using SSL, proceed anyway otherwise - CURLUSESSL_CONTROL - SSL for the control connection or fail - CURLUSESSL_ALL - SSL for all communication or fail - */ - CINIT(USE_SSL, LONG, 119), - - /* The _LARGE version of the standard POSTFIELDSIZE option */ - CINIT(POSTFIELDSIZE_LARGE, OFF_T, 120), - - /* Enable/disable the TCP Nagle algorithm */ - CINIT(TCP_NODELAY, LONG, 121), - - /* 122 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ - /* 123 OBSOLETE. Gone in 7.16.0 */ - /* 124 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ - /* 125 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ - /* 126 OBSOLETE, used in 7.12.3. Gone in 7.13.0 */ - /* 127 OBSOLETE. Gone in 7.16.0 */ - /* 128 OBSOLETE. Gone in 7.16.0 */ - - /* When FTP over SSL/TLS is selected (with CURLOPT_USE_SSL), this option - can be used to change libcurl's default action which is to first try - "AUTH SSL" and then "AUTH TLS" in this order, and proceed when a OK - response has been received. - - Available parameters are: - CURLFTPAUTH_DEFAULT - let libcurl decide - CURLFTPAUTH_SSL - try "AUTH SSL" first, then TLS - CURLFTPAUTH_TLS - try "AUTH TLS" first, then SSL - */ - CINIT(FTPSSLAUTH, LONG, 129), - - CINIT(IOCTLFUNCTION, FUNCTIONPOINT, 130), - CINIT(IOCTLDATA, OBJECTPOINT, 131), - - /* 132 OBSOLETE. Gone in 7.16.0 */ - /* 133 OBSOLETE. Gone in 7.16.0 */ - - /* zero terminated string for pass on to the FTP server when asked for - "account" info */ - CINIT(FTP_ACCOUNT, STRINGPOINT, 134), - - /* feed cookie into cookie engine */ - CINIT(COOKIELIST, STRINGPOINT, 135), - - /* ignore Content-Length */ - CINIT(IGNORE_CONTENT_LENGTH, LONG, 136), - - /* Set to non-zero to skip the IP address received in a 227 PASV FTP server - response. Typically used for FTP-SSL purposes but is not restricted to - that. libcurl will then instead use the same IP address it used for the - control connection. */ - CINIT(FTP_SKIP_PASV_IP, LONG, 137), - - /* Select "file method" to use when doing FTP, see the curl_ftpmethod - above. */ - CINIT(FTP_FILEMETHOD, LONG, 138), - - /* Local port number to bind the socket to */ - CINIT(LOCALPORT, LONG, 139), - - /* Number of ports to try, including the first one set with LOCALPORT. - Thus, setting it to 1 will make no additional attempts but the first. - */ - CINIT(LOCALPORTRANGE, LONG, 140), - - /* no transfer, set up connection and let application use the socket by - extracting it with CURLINFO_LASTSOCKET */ - CINIT(CONNECT_ONLY, LONG, 141), - - /* Function that will be called to convert from the - network encoding (instead of using the iconv calls in libcurl) */ - CINIT(CONV_FROM_NETWORK_FUNCTION, FUNCTIONPOINT, 142), - - /* Function that will be called to convert to the - network encoding (instead of using the iconv calls in libcurl) */ - CINIT(CONV_TO_NETWORK_FUNCTION, FUNCTIONPOINT, 143), - - /* Function that will be called to convert from UTF8 - (instead of using the iconv calls in libcurl) - Note that this is used only for SSL certificate processing */ - CINIT(CONV_FROM_UTF8_FUNCTION, FUNCTIONPOINT, 144), - - /* if the connection proceeds too quickly then need to slow it down */ - /* limit-rate: maximum number of bytes per second to send or receive */ - CINIT(MAX_SEND_SPEED_LARGE, OFF_T, 145), - CINIT(MAX_RECV_SPEED_LARGE, OFF_T, 146), - - /* Pointer to command string to send if USER/PASS fails. */ - CINIT(FTP_ALTERNATIVE_TO_USER, STRINGPOINT, 147), - - /* callback function for setting socket options */ - CINIT(SOCKOPTFUNCTION, FUNCTIONPOINT, 148), - CINIT(SOCKOPTDATA, OBJECTPOINT, 149), - - /* set to 0 to disable session ID re-use for this transfer, default is - enabled (== 1) */ - CINIT(SSL_SESSIONID_CACHE, LONG, 150), - - /* allowed SSH authentication methods */ - CINIT(SSH_AUTH_TYPES, LONG, 151), - - /* Used by scp/sftp to do public/private key authentication */ - CINIT(SSH_PUBLIC_KEYFILE, STRINGPOINT, 152), - CINIT(SSH_PRIVATE_KEYFILE, STRINGPOINT, 153), - - /* Send CCC (Clear Command Channel) after authentication */ - CINIT(FTP_SSL_CCC, LONG, 154), - - /* Same as TIMEOUT and CONNECTTIMEOUT, but with ms resolution */ - CINIT(TIMEOUT_MS, LONG, 155), - CINIT(CONNECTTIMEOUT_MS, LONG, 156), - - /* set to zero to disable the libcurl's decoding and thus pass the raw body - data to the application even when it is encoded/compressed */ - CINIT(HTTP_TRANSFER_DECODING, LONG, 157), - CINIT(HTTP_CONTENT_DECODING, LONG, 158), - - /* Permission used when creating new files and directories on the remote - server for protocols that support it, SFTP/SCP/FILE */ - CINIT(NEW_FILE_PERMS, LONG, 159), - CINIT(NEW_DIRECTORY_PERMS, LONG, 160), - - /* Set the behaviour of POST when redirecting. Values must be set to one - of CURL_REDIR* defines below. This used to be called CURLOPT_POST301 */ - CINIT(POSTREDIR, LONG, 161), - - /* used by scp/sftp to verify the host's public key */ - CINIT(SSH_HOST_PUBLIC_KEY_MD5, STRINGPOINT, 162), - - /* Callback function for opening socket (instead of socket(2)). Optionally, - callback is able change the address or refuse to connect returning - CURL_SOCKET_BAD. The callback should have type - curl_opensocket_callback */ - CINIT(OPENSOCKETFUNCTION, FUNCTIONPOINT, 163), - CINIT(OPENSOCKETDATA, OBJECTPOINT, 164), - - /* POST volatile input fields. */ - CINIT(COPYPOSTFIELDS, OBJECTPOINT, 165), - - /* set transfer mode (;type=) when doing FTP via an HTTP proxy */ - CINIT(PROXY_TRANSFER_MODE, LONG, 166), - - /* Callback function for seeking in the input stream */ - CINIT(SEEKFUNCTION, FUNCTIONPOINT, 167), - CINIT(SEEKDATA, OBJECTPOINT, 168), - - /* CRL file */ - CINIT(CRLFILE, STRINGPOINT, 169), - - /* Issuer certificate */ - CINIT(ISSUERCERT, STRINGPOINT, 170), - - /* (IPv6) Address scope */ - CINIT(ADDRESS_SCOPE, LONG, 171), - - /* Collect certificate chain info and allow it to get retrievable with - CURLINFO_CERTINFO after the transfer is complete. */ - CINIT(CERTINFO, LONG, 172), - - /* "name" and "pwd" to use when fetching. */ - CINIT(USERNAME, STRINGPOINT, 173), - CINIT(PASSWORD, STRINGPOINT, 174), - - /* "name" and "pwd" to use with Proxy when fetching. */ - CINIT(PROXYUSERNAME, STRINGPOINT, 175), - CINIT(PROXYPASSWORD, STRINGPOINT, 176), - - /* Comma separated list of hostnames defining no-proxy zones. These should - match both hostnames directly, and hostnames within a domain. For - example, local.com will match local.com and www.local.com, but NOT - notlocal.com or www.notlocal.com. For compatibility with other - implementations of this, .local.com will be considered to be the same as - local.com. A single * is the only valid wildcard, and effectively - disables the use of proxy. */ - CINIT(NOPROXY, STRINGPOINT, 177), - - /* block size for TFTP transfers */ - CINIT(TFTP_BLKSIZE, LONG, 178), - - /* Socks Service */ - CINIT(SOCKS5_GSSAPI_SERVICE, STRINGPOINT, 179), - - /* Socks Service */ - CINIT(SOCKS5_GSSAPI_NEC, LONG, 180), - - /* set the bitmask for the protocols that are allowed to be used for the - transfer, which thus helps the app which takes URLs from users or other - external inputs and want to restrict what protocol(s) to deal - with. Defaults to CURLPROTO_ALL. */ - CINIT(PROTOCOLS, LONG, 181), - - /* set the bitmask for the protocols that libcurl is allowed to follow to, - as a subset of the CURLOPT_PROTOCOLS ones. That means the protocol needs - to be set in both bitmasks to be allowed to get redirected to. Defaults - to all protocols except FILE and SCP. */ - CINIT(REDIR_PROTOCOLS, LONG, 182), - - /* set the SSH knownhost file name to use */ - CINIT(SSH_KNOWNHOSTS, STRINGPOINT, 183), - - /* set the SSH host key callback, must point to a curl_sshkeycallback - function */ - CINIT(SSH_KEYFUNCTION, FUNCTIONPOINT, 184), - - /* set the SSH host key callback custom pointer */ - CINIT(SSH_KEYDATA, OBJECTPOINT, 185), - - /* set the SMTP mail originator */ - CINIT(MAIL_FROM, STRINGPOINT, 186), - - /* set the list of SMTP mail receiver(s) */ - CINIT(MAIL_RCPT, OBJECTPOINT, 187), - - /* FTP: send PRET before PASV */ - CINIT(FTP_USE_PRET, LONG, 188), - - /* RTSP request method (OPTIONS, SETUP, PLAY, etc...) */ - CINIT(RTSP_REQUEST, LONG, 189), - - /* The RTSP session identifier */ - CINIT(RTSP_SESSION_ID, STRINGPOINT, 190), - - /* The RTSP stream URI */ - CINIT(RTSP_STREAM_URI, STRINGPOINT, 191), - - /* The Transport: header to use in RTSP requests */ - CINIT(RTSP_TRANSPORT, STRINGPOINT, 192), - - /* Manually initialize the client RTSP CSeq for this handle */ - CINIT(RTSP_CLIENT_CSEQ, LONG, 193), - - /* Manually initialize the server RTSP CSeq for this handle */ - CINIT(RTSP_SERVER_CSEQ, LONG, 194), - - /* The stream to pass to INTERLEAVEFUNCTION. */ - CINIT(INTERLEAVEDATA, OBJECTPOINT, 195), - - /* Let the application define a custom write method for RTP data */ - CINIT(INTERLEAVEFUNCTION, FUNCTIONPOINT, 196), - - /* Turn on wildcard matching */ - CINIT(WILDCARDMATCH, LONG, 197), - - /* Directory matching callback called before downloading of an - individual file (chunk) started */ - CINIT(CHUNK_BGN_FUNCTION, FUNCTIONPOINT, 198), - - /* Directory matching callback called after the file (chunk) - was downloaded, or skipped */ - CINIT(CHUNK_END_FUNCTION, FUNCTIONPOINT, 199), - - /* Change match (fnmatch-like) callback for wildcard matching */ - CINIT(FNMATCH_FUNCTION, FUNCTIONPOINT, 200), - - /* Let the application define custom chunk data pointer */ - CINIT(CHUNK_DATA, OBJECTPOINT, 201), - - /* FNMATCH_FUNCTION user pointer */ - CINIT(FNMATCH_DATA, OBJECTPOINT, 202), - - /* send linked-list of name:port:address sets */ - CINIT(RESOLVE, OBJECTPOINT, 203), - - /* Set a username for authenticated TLS */ - CINIT(TLSAUTH_USERNAME, STRINGPOINT, 204), - - /* Set a password for authenticated TLS */ - CINIT(TLSAUTH_PASSWORD, STRINGPOINT, 205), - - /* Set authentication type for authenticated TLS */ - CINIT(TLSAUTH_TYPE, STRINGPOINT, 206), - - /* Set to 1 to enable the "TE:" header in HTTP requests to ask for - compressed transfer-encoded responses. Set to 0 to disable the use of TE: - in outgoing requests. The current default is 0, but it might change in a - future libcurl release. - - libcurl will ask for the compressed methods it knows of, and if that - isn't any, it will not ask for transfer-encoding at all even if this - option is set to 1. - - */ - CINIT(TRANSFER_ENCODING, LONG, 207), - - /* Callback function for closing socket (instead of close(2)). The callback - should have type curl_closesocket_callback */ - CINIT(CLOSESOCKETFUNCTION, FUNCTIONPOINT, 208), - CINIT(CLOSESOCKETDATA, OBJECTPOINT, 209), - - /* allow GSSAPI credential delegation */ - CINIT(GSSAPI_DELEGATION, LONG, 210), - - /* Set the name servers to use for DNS resolution */ - CINIT(DNS_SERVERS, STRINGPOINT, 211), - - /* Time-out accept operations (currently for FTP only) after this amount - of miliseconds. */ - CINIT(ACCEPTTIMEOUT_MS, LONG, 212), - - /* Set TCP keepalive */ - CINIT(TCP_KEEPALIVE, LONG, 213), - - /* non-universal keepalive knobs (Linux, AIX, HP-UX, more) */ - CINIT(TCP_KEEPIDLE, LONG, 214), - CINIT(TCP_KEEPINTVL, LONG, 215), - - /* Enable/disable specific SSL features with a bitmask, see CURLSSLOPT_* */ - CINIT(SSL_OPTIONS, LONG, 216), - - /* Set the SMTP auth originator */ - CINIT(MAIL_AUTH, STRINGPOINT, 217), - - /* Enable/disable SASL initial response */ - CINIT(SASL_IR, LONG, 218), - - /* Function that will be called instead of the internal progress display - * function. This function should be defined as the curl_xferinfo_callback - * prototype defines. (Deprecates CURLOPT_PROGRESSFUNCTION) */ - CINIT(XFERINFOFUNCTION, FUNCTIONPOINT, 219), - - /* The XOAUTH2 bearer token */ - CINIT(XOAUTH2_BEARER, STRINGPOINT, 220), - - /* Set the interface string to use as outgoing network - * interface for DNS requests. - * Only supported by the c-ares DNS backend */ - CINIT(DNS_INTERFACE, STRINGPOINT, 221), - - /* Set the local IPv4 address to use for outgoing DNS requests. - * Only supported by the c-ares DNS backend */ - CINIT(DNS_LOCAL_IP4, STRINGPOINT, 222), - - /* Set the local IPv4 address to use for outgoing DNS requests. - * Only supported by the c-ares DNS backend */ - CINIT(DNS_LOCAL_IP6, STRINGPOINT, 223), - - /* Set authentication options directly */ - CINIT(LOGIN_OPTIONS, STRINGPOINT, 224), - - /* Enable/disable TLS NPN extension (http2 over ssl might fail without) */ - CINIT(SSL_ENABLE_NPN, LONG, 225), - - /* Enable/disable TLS ALPN extension (http2 over ssl might fail without) */ - CINIT(SSL_ENABLE_ALPN, LONG, 226), - - /* Time to wait for a response to a HTTP request containing an - * Expect: 100-continue header before sending the data anyway. */ - CINIT(EXPECT_100_TIMEOUT_MS, LONG, 227), - - /* This points to a linked list of headers used for proxy requests only, - struct curl_slist kind */ - CINIT(PROXYHEADER, OBJECTPOINT, 228), - - /* Pass in a bitmask of "header options" */ - CINIT(HEADEROPT, LONG, 229), - - /* The public key in DER form used to validate the peer public key - this option is used only if SSL_VERIFYPEER is true */ - CINIT(PINNEDPUBLICKEY, STRINGPOINT, 230), - - /* Path to Unix domain socket */ - CINIT(UNIX_SOCKET_PATH, STRINGPOINT, 231), - - /* Set if we should verify the certificate status. */ - CINIT(SSL_VERIFYSTATUS, LONG, 232), - - /* Set if we should enable TLS false start. */ - CINIT(SSL_FALSESTART, LONG, 233), - - /* Do not squash dot-dot sequences */ - CINIT(PATH_AS_IS, LONG, 234), - - /* Proxy Service Name */ - CINIT(PROXY_SERVICE_NAME, STRINGPOINT, 235), - - /* Service Name */ - CINIT(SERVICE_NAME, STRINGPOINT, 236), - - /* Wait/don't wait for pipe/mutex to clarify */ - CINIT(PIPEWAIT, LONG, 237), - - /* Set the protocol used when curl is given a URL without a protocol */ - CINIT(DEFAULT_PROTOCOL, STRINGPOINT, 238), - - /* Set stream weight, 1 - 256 (default is 16) */ - CINIT(STREAM_WEIGHT, LONG, 239), - - /* Set stream dependency on another CURL handle */ - CINIT(STREAM_DEPENDS, OBJECTPOINT, 240), - - /* Set E-xclusive stream dependency on another CURL handle */ - CINIT(STREAM_DEPENDS_E, OBJECTPOINT, 241), - - CURLOPT_LASTENTRY /* the last unused */ -} CURLoption; - -#ifndef CURL_NO_OLDIES /* define this to test if your app builds with all - the obsolete stuff removed! */ - -/* Backwards compatibility with older names */ -/* These are scheduled to disappear by 2011 */ - -/* This was added in version 7.19.1 */ -#define CURLOPT_POST301 CURLOPT_POSTREDIR - -/* These are scheduled to disappear by 2009 */ - -/* The following were added in 7.17.0 */ -#define CURLOPT_SSLKEYPASSWD CURLOPT_KEYPASSWD -#define CURLOPT_FTPAPPEND CURLOPT_APPEND -#define CURLOPT_FTPLISTONLY CURLOPT_DIRLISTONLY -#define CURLOPT_FTP_SSL CURLOPT_USE_SSL - -/* The following were added earlier */ - -#define CURLOPT_SSLCERTPASSWD CURLOPT_KEYPASSWD -#define CURLOPT_KRB4LEVEL CURLOPT_KRBLEVEL - -#else -/* This is set if CURL_NO_OLDIES is defined at compile-time */ -#undef CURLOPT_DNS_USE_GLOBAL_CACHE /* soon obsolete */ -#endif - - - /* Below here follows defines for the CURLOPT_IPRESOLVE option. If a host - name resolves addresses using more than one IP protocol version, this - option might be handy to force libcurl to use a specific IP version. */ -#define CURL_IPRESOLVE_WHATEVER 0 /* default, resolves addresses to all IP - versions that your system allows */ -#define CURL_IPRESOLVE_V4 1 /* resolve to IPv4 addresses */ -#define CURL_IPRESOLVE_V6 2 /* resolve to IPv6 addresses */ - - /* three convenient "aliases" that follow the name scheme better */ -#define CURLOPT_RTSPHEADER CURLOPT_HTTPHEADER - - /* These enums are for use with the CURLOPT_HTTP_VERSION option. */ -enum { - CURL_HTTP_VERSION_NONE, /* setting this means we don't care, and that we'd - like the library to choose the best possible - for us! */ - CURL_HTTP_VERSION_1_0, /* please use HTTP 1.0 in the request */ - CURL_HTTP_VERSION_1_1, /* please use HTTP 1.1 in the request */ - CURL_HTTP_VERSION_2_0, /* please use HTTP 2 in the request */ - CURL_HTTP_VERSION_2TLS, /* use version 2 for HTTPS, version 1.1 for HTTP */ - - CURL_HTTP_VERSION_LAST /* *ILLEGAL* http version */ -}; - -/* Convenience definition simple because the name of the version is HTTP/2 and - not 2.0. The 2_0 version of the enum name was set while the version was - still planned to be 2.0 and we stick to it for compatibility. */ -#define CURL_HTTP_VERSION_2 CURL_HTTP_VERSION_2_0 - -/* - * Public API enums for RTSP requests - */ -enum { - CURL_RTSPREQ_NONE, /* first in list */ - CURL_RTSPREQ_OPTIONS, - CURL_RTSPREQ_DESCRIBE, - CURL_RTSPREQ_ANNOUNCE, - CURL_RTSPREQ_SETUP, - CURL_RTSPREQ_PLAY, - CURL_RTSPREQ_PAUSE, - CURL_RTSPREQ_TEARDOWN, - CURL_RTSPREQ_GET_PARAMETER, - CURL_RTSPREQ_SET_PARAMETER, - CURL_RTSPREQ_RECORD, - CURL_RTSPREQ_RECEIVE, - CURL_RTSPREQ_LAST /* last in list */ -}; - - /* These enums are for use with the CURLOPT_NETRC option. */ -enum CURL_NETRC_OPTION { - CURL_NETRC_IGNORED, /* The .netrc will never be read. - * This is the default. */ - CURL_NETRC_OPTIONAL, /* A user:password in the URL will be preferred - * to one in the .netrc. */ - CURL_NETRC_REQUIRED, /* A user:password in the URL will be ignored. - * Unless one is set programmatically, the .netrc - * will be queried. */ - CURL_NETRC_LAST -}; - -enum { - CURL_SSLVERSION_DEFAULT, - CURL_SSLVERSION_TLSv1, /* TLS 1.x */ - CURL_SSLVERSION_SSLv2, - CURL_SSLVERSION_SSLv3, - CURL_SSLVERSION_TLSv1_0, - CURL_SSLVERSION_TLSv1_1, - CURL_SSLVERSION_TLSv1_2, - - CURL_SSLVERSION_LAST /* never use, keep last */ -}; - -enum CURL_TLSAUTH { - CURL_TLSAUTH_NONE, - CURL_TLSAUTH_SRP, - CURL_TLSAUTH_LAST /* never use, keep last */ -}; - -/* symbols to use with CURLOPT_POSTREDIR. - CURL_REDIR_POST_301, CURL_REDIR_POST_302 and CURL_REDIR_POST_303 - can be bitwise ORed so that CURL_REDIR_POST_301 | CURL_REDIR_POST_302 - | CURL_REDIR_POST_303 == CURL_REDIR_POST_ALL */ - -#define CURL_REDIR_GET_ALL 0 -#define CURL_REDIR_POST_301 1 -#define CURL_REDIR_POST_302 2 -#define CURL_REDIR_POST_303 4 -#define CURL_REDIR_POST_ALL \ - (CURL_REDIR_POST_301|CURL_REDIR_POST_302|CURL_REDIR_POST_303) - -typedef enum { - CURL_TIMECOND_NONE, - - CURL_TIMECOND_IFMODSINCE, - CURL_TIMECOND_IFUNMODSINCE, - CURL_TIMECOND_LASTMOD, - - CURL_TIMECOND_LAST -} curl_TimeCond; - - -/* curl_strequal() and curl_strnequal() are subject for removal in a future - libcurl, see lib/README.curlx for details */ -CURL_EXTERN int (curl_strequal)(const char *s1, const char *s2); -CURL_EXTERN int (curl_strnequal)(const char *s1, const char *s2, size_t n); - -/* name is uppercase CURLFORM_ */ -#ifdef CFINIT -#undef CFINIT -#endif - -#ifdef CURL_ISOCPP -#define CFINIT(name) CURLFORM_ ## name -#else -/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ -#define CFINIT(name) CURLFORM_/**/name -#endif - -typedef enum { - CFINIT(NOTHING), /********* the first one is unused ************/ - - /* */ - CFINIT(COPYNAME), - CFINIT(PTRNAME), - CFINIT(NAMELENGTH), - CFINIT(COPYCONTENTS), - CFINIT(PTRCONTENTS), - CFINIT(CONTENTSLENGTH), - CFINIT(FILECONTENT), - CFINIT(ARRAY), - CFINIT(OBSOLETE), - CFINIT(FILE), - - CFINIT(BUFFER), - CFINIT(BUFFERPTR), - CFINIT(BUFFERLENGTH), - - CFINIT(CONTENTTYPE), - CFINIT(CONTENTHEADER), - CFINIT(FILENAME), - CFINIT(END), - CFINIT(OBSOLETE2), - - CFINIT(STREAM), - CFINIT(CONTENTLEN), /* added in 7.46.0, provide a curl_off_t length */ - - CURLFORM_LASTENTRY /* the last unused */ -} CURLformoption; - -#undef CFINIT /* done */ - -/* structure to be used as parameter for CURLFORM_ARRAY */ -struct curl_forms { - CURLformoption option; - const char *value; -}; - -/* use this for multipart formpost building */ -/* Returns code for curl_formadd() - * - * Returns: - * CURL_FORMADD_OK on success - * CURL_FORMADD_MEMORY if the FormInfo allocation fails - * CURL_FORMADD_OPTION_TWICE if one option is given twice for one Form - * CURL_FORMADD_NULL if a null pointer was given for a char - * CURL_FORMADD_MEMORY if the allocation of a FormInfo struct failed - * CURL_FORMADD_UNKNOWN_OPTION if an unknown option was used - * CURL_FORMADD_INCOMPLETE if the some FormInfo is not complete (or error) - * CURL_FORMADD_MEMORY if a curl_httppost struct cannot be allocated - * CURL_FORMADD_MEMORY if some allocation for string copying failed. - * CURL_FORMADD_ILLEGAL_ARRAY if an illegal option is used in an array - * - ***************************************************************************/ -typedef enum { - CURL_FORMADD_OK, /* first, no error */ - - CURL_FORMADD_MEMORY, - CURL_FORMADD_OPTION_TWICE, - CURL_FORMADD_NULL, - CURL_FORMADD_UNKNOWN_OPTION, - CURL_FORMADD_INCOMPLETE, - CURL_FORMADD_ILLEGAL_ARRAY, - CURL_FORMADD_DISABLED, /* libcurl was built with this disabled */ - - CURL_FORMADD_LAST /* last */ -} CURLFORMcode; - -/* - * NAME curl_formadd() - * - * DESCRIPTION - * - * Pretty advanced function for building multi-part formposts. Each invoke - * adds one part that together construct a full post. Then use - * CURLOPT_HTTPPOST to send it off to libcurl. - */ -CURL_EXTERN CURLFORMcode curl_formadd(struct curl_httppost **httppost, - struct curl_httppost **last_post, - ...); - -/* - * callback function for curl_formget() - * The void *arg pointer will be the one passed as second argument to - * curl_formget(). - * The character buffer passed to it must not be freed. - * Should return the buffer length passed to it as the argument "len" on - * success. - */ -typedef size_t (*curl_formget_callback)(void *arg, const char *buf, - size_t len); - -/* - * NAME curl_formget() - * - * DESCRIPTION - * - * Serialize a curl_httppost struct built with curl_formadd(). - * Accepts a void pointer as second argument which will be passed to - * the curl_formget_callback function. - * Returns 0 on success. - */ -CURL_EXTERN int curl_formget(struct curl_httppost *form, void *arg, - curl_formget_callback append); -/* - * NAME curl_formfree() - * - * DESCRIPTION - * - * Free a multipart formpost previously built with curl_formadd(). - */ -CURL_EXTERN void curl_formfree(struct curl_httppost *form); - -/* - * NAME curl_getenv() - * - * DESCRIPTION - * - * Returns a malloc()'ed string that MUST be curl_free()ed after usage is - * complete. DEPRECATED - see lib/README.curlx - */ -CURL_EXTERN char *curl_getenv(const char *variable); - -/* - * NAME curl_version() - * - * DESCRIPTION - * - * Returns a static ascii string of the libcurl version. - */ -CURL_EXTERN char *curl_version(void); - -/* - * NAME curl_easy_escape() - * - * DESCRIPTION - * - * Escapes URL strings (converts all letters consider illegal in URLs to their - * %XX versions). This function returns a new allocated string or NULL if an - * error occurred. - */ -CURL_EXTERN char *curl_easy_escape(CURL *handle, - const char *string, - int length); - -/* the previous version: */ -CURL_EXTERN char *curl_escape(const char *string, - int length); - - -/* - * NAME curl_easy_unescape() - * - * DESCRIPTION - * - * Unescapes URL encoding in strings (converts all %XX codes to their 8bit - * versions). This function returns a new allocated string or NULL if an error - * occurred. - * Conversion Note: On non-ASCII platforms the ASCII %XX codes are - * converted into the host encoding. - */ -CURL_EXTERN char *curl_easy_unescape(CURL *handle, - const char *string, - int length, - int *outlength); - -/* the previous version */ -CURL_EXTERN char *curl_unescape(const char *string, - int length); - -/* - * NAME curl_free() - * - * DESCRIPTION - * - * Provided for de-allocation in the same translation unit that did the - * allocation. Added in libcurl 7.10 - */ -CURL_EXTERN void curl_free(void *p); - -/* - * NAME curl_global_init() - * - * DESCRIPTION - * - * curl_global_init() should be invoked exactly once for each application that - * uses libcurl and before any call of other libcurl functions. - * - * This function is not thread-safe! - */ -CURL_EXTERN CURLcode curl_global_init(long flags); - -/* - * NAME curl_global_init_mem() - * - * DESCRIPTION - * - * curl_global_init() or curl_global_init_mem() should be invoked exactly once - * for each application that uses libcurl. This function can be used to - * initialize libcurl and set user defined memory management callback - * functions. Users can implement memory management routines to check for - * memory leaks, check for mis-use of the curl library etc. User registered - * callback routines with be invoked by this library instead of the system - * memory management routines like malloc, free etc. - */ -CURL_EXTERN CURLcode curl_global_init_mem(long flags, - curl_malloc_callback m, - curl_free_callback f, - curl_realloc_callback r, - curl_strdup_callback s, - curl_calloc_callback c); - -/* - * NAME curl_global_cleanup() - * - * DESCRIPTION - * - * curl_global_cleanup() should be invoked exactly once for each application - * that uses libcurl - */ -CURL_EXTERN void curl_global_cleanup(void); - -/* linked-list structure for the CURLOPT_QUOTE option (and other) */ -struct curl_slist { - char *data; - struct curl_slist *next; -}; - -/* - * NAME curl_slist_append() - * - * DESCRIPTION - * - * Appends a string to a linked list. If no list exists, it will be created - * first. Returns the new list, after appending. - */ -CURL_EXTERN struct curl_slist *curl_slist_append(struct curl_slist *, - const char *); - -/* - * NAME curl_slist_free_all() - * - * DESCRIPTION - * - * free a previously built curl_slist. - */ -CURL_EXTERN void curl_slist_free_all(struct curl_slist *); - -/* - * NAME curl_getdate() - * - * DESCRIPTION - * - * Returns the time, in seconds since 1 Jan 1970 of the time string given in - * the first argument. The time argument in the second parameter is unused - * and should be set to NULL. - */ -CURL_EXTERN time_t curl_getdate(const char *p, const time_t *unused); - -/* info about the certificate chain, only for OpenSSL builds. Asked - for with CURLOPT_CERTINFO / CURLINFO_CERTINFO */ -struct curl_certinfo { - int num_of_certs; /* number of certificates with information */ - struct curl_slist **certinfo; /* for each index in this array, there's a - linked list with textual information in the - format "name: value" */ -}; - -/* enum for the different supported SSL backends */ -typedef enum { - CURLSSLBACKEND_NONE = 0, - CURLSSLBACKEND_OPENSSL = 1, - CURLSSLBACKEND_GNUTLS = 2, - CURLSSLBACKEND_NSS = 3, - CURLSSLBACKEND_OBSOLETE4 = 4, /* Was QSOSSL. */ - CURLSSLBACKEND_GSKIT = 5, - CURLSSLBACKEND_POLARSSL = 6, - CURLSSLBACKEND_CYASSL = 7, - CURLSSLBACKEND_SCHANNEL = 8, - CURLSSLBACKEND_DARWINSSL = 9, - CURLSSLBACKEND_AXTLS = 10, - CURLSSLBACKEND_MBEDTLS = 11 -} curl_sslbackend; - -/* Information about the SSL library used and the respective internal SSL - handle, which can be used to obtain further information regarding the - connection. Asked for with CURLINFO_TLS_SESSION. */ -struct curl_tlssessioninfo { - curl_sslbackend backend; - void *internals; -}; - -#define CURLINFO_STRING 0x100000 -#define CURLINFO_LONG 0x200000 -#define CURLINFO_DOUBLE 0x300000 -#define CURLINFO_SLIST 0x400000 -#define CURLINFO_SOCKET 0x500000 -#define CURLINFO_MASK 0x0fffff -#define CURLINFO_TYPEMASK 0xf00000 - -typedef enum { - CURLINFO_NONE, /* first, never use this */ - CURLINFO_EFFECTIVE_URL = CURLINFO_STRING + 1, - CURLINFO_RESPONSE_CODE = CURLINFO_LONG + 2, - CURLINFO_TOTAL_TIME = CURLINFO_DOUBLE + 3, - CURLINFO_NAMELOOKUP_TIME = CURLINFO_DOUBLE + 4, - CURLINFO_CONNECT_TIME = CURLINFO_DOUBLE + 5, - CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6, - CURLINFO_SIZE_UPLOAD = CURLINFO_DOUBLE + 7, - CURLINFO_SIZE_DOWNLOAD = CURLINFO_DOUBLE + 8, - CURLINFO_SPEED_DOWNLOAD = CURLINFO_DOUBLE + 9, - CURLINFO_SPEED_UPLOAD = CURLINFO_DOUBLE + 10, - CURLINFO_HEADER_SIZE = CURLINFO_LONG + 11, - CURLINFO_REQUEST_SIZE = CURLINFO_LONG + 12, - CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG + 13, - CURLINFO_FILETIME = CURLINFO_LONG + 14, - CURLINFO_CONTENT_LENGTH_DOWNLOAD = CURLINFO_DOUBLE + 15, - CURLINFO_CONTENT_LENGTH_UPLOAD = CURLINFO_DOUBLE + 16, - CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17, - CURLINFO_CONTENT_TYPE = CURLINFO_STRING + 18, - CURLINFO_REDIRECT_TIME = CURLINFO_DOUBLE + 19, - CURLINFO_REDIRECT_COUNT = CURLINFO_LONG + 20, - CURLINFO_PRIVATE = CURLINFO_STRING + 21, - CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG + 22, - CURLINFO_HTTPAUTH_AVAIL = CURLINFO_LONG + 23, - CURLINFO_PROXYAUTH_AVAIL = CURLINFO_LONG + 24, - CURLINFO_OS_ERRNO = CURLINFO_LONG + 25, - CURLINFO_NUM_CONNECTS = CURLINFO_LONG + 26, - CURLINFO_SSL_ENGINES = CURLINFO_SLIST + 27, - CURLINFO_COOKIELIST = CURLINFO_SLIST + 28, - CURLINFO_LASTSOCKET = CURLINFO_LONG + 29, - CURLINFO_FTP_ENTRY_PATH = CURLINFO_STRING + 30, - CURLINFO_REDIRECT_URL = CURLINFO_STRING + 31, - CURLINFO_PRIMARY_IP = CURLINFO_STRING + 32, - CURLINFO_APPCONNECT_TIME = CURLINFO_DOUBLE + 33, - CURLINFO_CERTINFO = CURLINFO_SLIST + 34, - CURLINFO_CONDITION_UNMET = CURLINFO_LONG + 35, - CURLINFO_RTSP_SESSION_ID = CURLINFO_STRING + 36, - CURLINFO_RTSP_CLIENT_CSEQ = CURLINFO_LONG + 37, - CURLINFO_RTSP_SERVER_CSEQ = CURLINFO_LONG + 38, - CURLINFO_RTSP_CSEQ_RECV = CURLINFO_LONG + 39, - CURLINFO_PRIMARY_PORT = CURLINFO_LONG + 40, - CURLINFO_LOCAL_IP = CURLINFO_STRING + 41, - CURLINFO_LOCAL_PORT = CURLINFO_LONG + 42, - CURLINFO_TLS_SESSION = CURLINFO_SLIST + 43, - CURLINFO_ACTIVESOCKET = CURLINFO_SOCKET + 44, - /* Fill in new entries below here! */ - - CURLINFO_LASTONE = 44 -} CURLINFO; - -/* CURLINFO_RESPONSE_CODE is the new name for the option previously known as - CURLINFO_HTTP_CODE */ -#define CURLINFO_HTTP_CODE CURLINFO_RESPONSE_CODE - -typedef enum { - CURLCLOSEPOLICY_NONE, /* first, never use this */ - - CURLCLOSEPOLICY_OLDEST, - CURLCLOSEPOLICY_LEAST_RECENTLY_USED, - CURLCLOSEPOLICY_LEAST_TRAFFIC, - CURLCLOSEPOLICY_SLOWEST, - CURLCLOSEPOLICY_CALLBACK, - - CURLCLOSEPOLICY_LAST /* last, never use this */ -} curl_closepolicy; - -#define CURL_GLOBAL_SSL (1<<0) -#define CURL_GLOBAL_WIN32 (1<<1) -#define CURL_GLOBAL_ALL (CURL_GLOBAL_SSL|CURL_GLOBAL_WIN32) -#define CURL_GLOBAL_NOTHING 0 -#define CURL_GLOBAL_DEFAULT CURL_GLOBAL_ALL -#define CURL_GLOBAL_ACK_EINTR (1<<2) - - -/***************************************************************************** - * Setup defines, protos etc for the sharing stuff. - */ - -/* Different data locks for a single share */ -typedef enum { - CURL_LOCK_DATA_NONE = 0, - /* CURL_LOCK_DATA_SHARE is used internally to say that - * the locking is just made to change the internal state of the share - * itself. - */ - CURL_LOCK_DATA_SHARE, - CURL_LOCK_DATA_COOKIE, - CURL_LOCK_DATA_DNS, - CURL_LOCK_DATA_SSL_SESSION, - CURL_LOCK_DATA_CONNECT, - CURL_LOCK_DATA_LAST -} curl_lock_data; - -/* Different lock access types */ -typedef enum { - CURL_LOCK_ACCESS_NONE = 0, /* unspecified action */ - CURL_LOCK_ACCESS_SHARED = 1, /* for read perhaps */ - CURL_LOCK_ACCESS_SINGLE = 2, /* for write perhaps */ - CURL_LOCK_ACCESS_LAST /* never use */ -} curl_lock_access; - -typedef void (*curl_lock_function)(CURL *handle, - curl_lock_data data, - curl_lock_access locktype, - void *userptr); -typedef void (*curl_unlock_function)(CURL *handle, - curl_lock_data data, - void *userptr); - -typedef void CURLSH; - -typedef enum { - CURLSHE_OK, /* all is fine */ - CURLSHE_BAD_OPTION, /* 1 */ - CURLSHE_IN_USE, /* 2 */ - CURLSHE_INVALID, /* 3 */ - CURLSHE_NOMEM, /* 4 out of memory */ - CURLSHE_NOT_BUILT_IN, /* 5 feature not present in lib */ - CURLSHE_LAST /* never use */ -} CURLSHcode; - -typedef enum { - CURLSHOPT_NONE, /* don't use */ - CURLSHOPT_SHARE, /* specify a data type to share */ - CURLSHOPT_UNSHARE, /* specify which data type to stop sharing */ - CURLSHOPT_LOCKFUNC, /* pass in a 'curl_lock_function' pointer */ - CURLSHOPT_UNLOCKFUNC, /* pass in a 'curl_unlock_function' pointer */ - CURLSHOPT_USERDATA, /* pass in a user data pointer used in the lock/unlock - callback functions */ - CURLSHOPT_LAST /* never use */ -} CURLSHoption; - -CURL_EXTERN CURLSH *curl_share_init(void); -CURL_EXTERN CURLSHcode curl_share_setopt(CURLSH *, CURLSHoption option, ...); -CURL_EXTERN CURLSHcode curl_share_cleanup(CURLSH *); - -/**************************************************************************** - * Structures for querying information about the curl library at runtime. - */ - -typedef enum { - CURLVERSION_FIRST, - CURLVERSION_SECOND, - CURLVERSION_THIRD, - CURLVERSION_FOURTH, - CURLVERSION_LAST /* never actually use this */ -} CURLversion; - -/* The 'CURLVERSION_NOW' is the symbolic name meant to be used by - basically all programs ever that want to get version information. It is - meant to be a built-in version number for what kind of struct the caller - expects. If the struct ever changes, we redefine the NOW to another enum - from above. */ -#define CURLVERSION_NOW CURLVERSION_FOURTH - -typedef struct { - CURLversion age; /* age of the returned struct */ - const char *version; /* LIBCURL_VERSION */ - unsigned int version_num; /* LIBCURL_VERSION_NUM */ - const char *host; /* OS/host/cpu/machine when configured */ - int features; /* bitmask, see defines below */ - const char *ssl_version; /* human readable string */ - long ssl_version_num; /* not used anymore, always 0 */ - const char *libz_version; /* human readable string */ - /* protocols is terminated by an entry with a NULL protoname */ - const char * const *protocols; - - /* The fields below this were added in CURLVERSION_SECOND */ - const char *ares; - int ares_num; - - /* This field was added in CURLVERSION_THIRD */ - const char *libidn; - - /* These field were added in CURLVERSION_FOURTH */ - - /* Same as '_libiconv_version' if built with HAVE_ICONV */ - int iconv_ver_num; - - const char *libssh_version; /* human readable string */ - -} curl_version_info_data; - -#define CURL_VERSION_IPV6 (1<<0) /* IPv6-enabled */ -#define CURL_VERSION_KERBEROS4 (1<<1) /* Kerberos V4 auth is supported - (deprecated) */ -#define CURL_VERSION_SSL (1<<2) /* SSL options are present */ -#define CURL_VERSION_LIBZ (1<<3) /* libz features are present */ -#define CURL_VERSION_NTLM (1<<4) /* NTLM auth is supported */ -#define CURL_VERSION_GSSNEGOTIATE (1<<5) /* Negotiate auth is supported - (deprecated) */ -#define CURL_VERSION_DEBUG (1<<6) /* Built with debug capabilities */ -#define CURL_VERSION_ASYNCHDNS (1<<7) /* Asynchronous DNS resolves */ -#define CURL_VERSION_SPNEGO (1<<8) /* SPNEGO auth is supported */ -#define CURL_VERSION_LARGEFILE (1<<9) /* Supports files larger than 2GB */ -#define CURL_VERSION_IDN (1<<10) /* Internationized Domain Names are - supported */ -#define CURL_VERSION_SSPI (1<<11) /* Built against Windows SSPI */ -#define CURL_VERSION_CONV (1<<12) /* Character conversions supported */ -#define CURL_VERSION_CURLDEBUG (1<<13) /* Debug memory tracking supported */ -#define CURL_VERSION_TLSAUTH_SRP (1<<14) /* TLS-SRP auth is supported */ -#define CURL_VERSION_NTLM_WB (1<<15) /* NTLM delegation to winbind helper - is suported */ -#define CURL_VERSION_HTTP2 (1<<16) /* HTTP2 support built-in */ -#define CURL_VERSION_GSSAPI (1<<17) /* Built against a GSS-API library */ -#define CURL_VERSION_KERBEROS5 (1<<18) /* Kerberos V5 auth is supported */ -#define CURL_VERSION_UNIX_SOCKETS (1<<19) /* Unix domain sockets support */ -#define CURL_VERSION_PSL (1<<20) /* Mozilla's Public Suffix List, used - for cookie domain verification */ - - /* - * NAME curl_version_info() - * - * DESCRIPTION - * - * This function returns a pointer to a static copy of the version info - * struct. See above. - */ -CURL_EXTERN curl_version_info_data *curl_version_info(CURLversion); - -/* - * NAME curl_easy_strerror() - * - * DESCRIPTION - * - * The curl_easy_strerror function may be used to turn a CURLcode value - * into the equivalent human readable error string. This is useful - * for printing meaningful error messages. - */ -CURL_EXTERN const char *curl_easy_strerror(CURLcode); - -/* - * NAME curl_share_strerror() - * - * DESCRIPTION - * - * The curl_share_strerror function may be used to turn a CURLSHcode value - * into the equivalent human readable error string. This is useful - * for printing meaningful error messages. - */ -CURL_EXTERN const char *curl_share_strerror(CURLSHcode); - -/* - * NAME curl_easy_pause() - * - * DESCRIPTION - * - * The curl_easy_pause function pauses or unpauses transfers. Select the new - * state by setting the bitmask, use the convenience defines below. - * - */ -CURL_EXTERN CURLcode curl_easy_pause(CURL *handle, int bitmask); - -#define CURLPAUSE_RECV (1<<0) -#define CURLPAUSE_RECV_CONT (0) - -#define CURLPAUSE_SEND (1<<2) -#define CURLPAUSE_SEND_CONT (0) - -#define CURLPAUSE_ALL (CURLPAUSE_RECV|CURLPAUSE_SEND) -#define CURLPAUSE_CONT (CURLPAUSE_RECV_CONT|CURLPAUSE_SEND_CONT) - -#ifdef __cplusplus -} -#endif - -/* unfortunately, the easy.h and multi.h include files need options and info - stuff before they can be included! */ -#include "easy.h" /* nothing in curl is fun without the easy stuff */ -#include "multi.h" - -/* the typechecker doesn't work in C++ (yet) */ -#if defined(__GNUC__) && defined(__GNUC_MINOR__) && \ - ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) && \ - !defined(__cplusplus) && !defined(CURL_DISABLE_TYPECHECK) -#include "typecheck-gcc.h" -#else -#if defined(__STDC__) && (__STDC__ >= 1) -/* This preprocessor magic that replaces a call with the exact same call is - only done to make sure application authors pass exactly three arguments - to these functions. */ -#define curl_easy_setopt(handle,opt,param) curl_easy_setopt(handle,opt,param) -#define curl_easy_getinfo(handle,info,arg) curl_easy_getinfo(handle,info,arg) -#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) -#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) -#endif /* __STDC__ >= 1 */ -#endif /* gcc >= 4.3 && !__cplusplus */ - -#endif /* __CURL_CURL_H */ diff --git a/deps/libcurl/include/curl/curlbuild.h b/deps/libcurl/include/curl/curlbuild.h deleted file mode 100644 index bdca52b5355feec5f80f279bea6c5799bcf1041e..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/curlbuild.h +++ /dev/null @@ -1,198 +0,0 @@ -/* include/curl/curlbuild.h. Generated from curlbuild.h.in by configure. */ -#ifndef __CURL_CURLBUILD_H -#define __CURL_CURLBUILD_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ - -/* ================================================================ */ -/* NOTES FOR CONFIGURE CAPABLE SYSTEMS */ -/* ================================================================ */ - -/* - * NOTE 1: - * ------- - * - * Nothing in this file is intended to be modified or adjusted by the - * curl library user nor by the curl library builder. - * - * If you think that something actually needs to be changed, adjusted - * or fixed in this file, then, report it on the libcurl development - * mailing list: http://cool.haxx.se/mailman/listinfo/curl-library/ - * - * This header file shall only export symbols which are 'curl' or 'CURL' - * prefixed, otherwise public name space would be polluted. - * - * NOTE 2: - * ------- - * - * Right now you might be staring at file include/curl/curlbuild.h.in or - * at file include/curl/curlbuild.h, this is due to the following reason: - * - * On systems capable of running the configure script, the configure process - * will overwrite the distributed include/curl/curlbuild.h file with one that - * is suitable and specific to the library being configured and built, which - * is generated from the include/curl/curlbuild.h.in template file. - * - */ - -/* ================================================================ */ -/* DEFINITION OF THESE SYMBOLS SHALL NOT TAKE PLACE ANYWHERE ELSE */ -/* ================================================================ */ - -#ifdef CURL_SIZEOF_LONG -#error "CURL_SIZEOF_LONG shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_SIZEOF_LONG_already_defined -#endif - -#ifdef CURL_TYPEOF_CURL_SOCKLEN_T -#error "CURL_TYPEOF_CURL_SOCKLEN_T shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_already_defined -#endif - -#ifdef CURL_SIZEOF_CURL_SOCKLEN_T -#error "CURL_SIZEOF_CURL_SOCKLEN_T shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_already_defined -#endif - -#ifdef CURL_TYPEOF_CURL_OFF_T -#error "CURL_TYPEOF_CURL_OFF_T shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_already_defined -#endif - -#ifdef CURL_FORMAT_CURL_OFF_T -#error "CURL_FORMAT_CURL_OFF_T shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_already_defined -#endif - -#ifdef CURL_FORMAT_CURL_OFF_TU -#error "CURL_FORMAT_CURL_OFF_TU shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_already_defined -#endif - -#ifdef CURL_FORMAT_OFF_T -#error "CURL_FORMAT_OFF_T shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_FORMAT_OFF_T_already_defined -#endif - -#ifdef CURL_SIZEOF_CURL_OFF_T -#error "CURL_SIZEOF_CURL_OFF_T shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_already_defined -#endif - -#ifdef CURL_SUFFIX_CURL_OFF_T -#error "CURL_SUFFIX_CURL_OFF_T shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_already_defined -#endif - -#ifdef CURL_SUFFIX_CURL_OFF_TU -#error "CURL_SUFFIX_CURL_OFF_TU shall not be defined except in curlbuild.h" - Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_already_defined -#endif - -/* ================================================================ */ -/* EXTERNAL INTERFACE SETTINGS FOR CONFIGURE CAPABLE SYSTEMS ONLY */ -/* ================================================================ */ - -/* Configure process defines this to 1 when it finds out that system */ -/* header file ws2tcpip.h must be included by the external interface. */ -/* #undef CURL_PULL_WS2TCPIP_H */ -#ifdef CURL_PULL_WS2TCPIP_H -# ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -# endif -# include -# include -# include -#endif - -/* Configure process defines this to 1 when it finds out that system */ -/* header file sys/types.h must be included by the external interface. */ -#define CURL_PULL_SYS_TYPES_H 1 -#ifdef CURL_PULL_SYS_TYPES_H -# include -#endif - -/* Configure process defines this to 1 when it finds out that system */ -/* header file stdint.h must be included by the external interface. */ -/* #undef CURL_PULL_STDINT_H */ -#ifdef CURL_PULL_STDINT_H -# include -#endif - -/* Configure process defines this to 1 when it finds out that system */ -/* header file inttypes.h must be included by the external interface. */ -/* #undef CURL_PULL_INTTYPES_H */ -#ifdef CURL_PULL_INTTYPES_H -# include -#endif - -/* Configure process defines this to 1 when it finds out that system */ -/* header file sys/socket.h must be included by the external interface. */ -#define CURL_PULL_SYS_SOCKET_H 1 -#ifdef CURL_PULL_SYS_SOCKET_H -# include -#endif - -/* Configure process defines this to 1 when it finds out that system */ -/* header file sys/poll.h must be included by the external interface. */ -/* #undef CURL_PULL_SYS_POLL_H */ -#ifdef CURL_PULL_SYS_POLL_H -# include -#endif - -/* The size of `long', as computed by sizeof. */ -#define CURL_SIZEOF_LONG 8 - -/* Integral data type used for curl_socklen_t. */ -#define CURL_TYPEOF_CURL_SOCKLEN_T socklen_t - -/* The size of `curl_socklen_t', as computed by sizeof. */ -#define CURL_SIZEOF_CURL_SOCKLEN_T 4 - -/* Data type definition of curl_socklen_t. */ -typedef CURL_TYPEOF_CURL_SOCKLEN_T curl_socklen_t; - -/* Signed integral data type used for curl_off_t. */ -#define CURL_TYPEOF_CURL_OFF_T long - -/* Data type definition of curl_off_t. */ -typedef CURL_TYPEOF_CURL_OFF_T curl_off_t; - -/* curl_off_t formatting string directive without "%" conversion specifier. */ -#define CURL_FORMAT_CURL_OFF_T "ld" - -/* unsigned curl_off_t formatting string without "%" conversion specifier. */ -#define CURL_FORMAT_CURL_OFF_TU "lu" - -/* curl_off_t formatting string directive with "%" conversion specifier. */ -#define CURL_FORMAT_OFF_T "%ld" - -/* The size of `curl_off_t', as computed by sizeof. */ -#define CURL_SIZEOF_CURL_OFF_T 8 - -/* curl_off_t constant suffix. */ -#define CURL_SUFFIX_CURL_OFF_T L - -/* unsigned curl_off_t constant suffix. */ -#define CURL_SUFFIX_CURL_OFF_TU UL - -#endif /* __CURL_CURLBUILD_H */ diff --git a/deps/libcurl/include/curl/curlrules.h b/deps/libcurl/include/curl/curlrules.h deleted file mode 100644 index 7c2ede35b63a7540acd042ce5c2b27a70c982d02..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/curlrules.h +++ /dev/null @@ -1,262 +0,0 @@ -#ifndef __CURL_CURLRULES_H -#define __CURL_CURLRULES_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2012, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ - -/* ================================================================ */ -/* COMPILE TIME SANITY CHECKS */ -/* ================================================================ */ - -/* - * NOTE 1: - * ------- - * - * All checks done in this file are intentionally placed in a public - * header file which is pulled by curl/curl.h when an application is - * being built using an already built libcurl library. Additionally - * this file is also included and used when building the library. - * - * If compilation fails on this file it is certainly sure that the - * problem is elsewhere. It could be a problem in the curlbuild.h - * header file, or simply that you are using different compilation - * settings than those used to build the library. - * - * Nothing in this file is intended to be modified or adjusted by the - * curl library user nor by the curl library builder. - * - * Do not deactivate any check, these are done to make sure that the - * library is properly built and used. - * - * You can find further help on the libcurl development mailing list: - * http://cool.haxx.se/mailman/listinfo/curl-library/ - * - * NOTE 2 - * ------ - * - * Some of the following compile time checks are based on the fact - * that the dimension of a constant array can not be a negative one. - * In this way if the compile time verification fails, the compilation - * will fail issuing an error. The error description wording is compiler - * dependent but it will be quite similar to one of the following: - * - * "negative subscript or subscript is too large" - * "array must have at least one element" - * "-1 is an illegal array size" - * "size of array is negative" - * - * If you are building an application which tries to use an already - * built libcurl library and you are getting this kind of errors on - * this file, it is a clear indication that there is a mismatch between - * how the library was built and how you are trying to use it for your - * application. Your already compiled or binary library provider is the - * only one who can give you the details you need to properly use it. - */ - -/* - * Verify that some macros are actually defined. - */ - -#ifndef CURL_SIZEOF_LONG -# error "CURL_SIZEOF_LONG definition is missing!" - Error Compilation_aborted_CURL_SIZEOF_LONG_is_missing -#endif - -#ifndef CURL_TYPEOF_CURL_SOCKLEN_T -# error "CURL_TYPEOF_CURL_SOCKLEN_T definition is missing!" - Error Compilation_aborted_CURL_TYPEOF_CURL_SOCKLEN_T_is_missing -#endif - -#ifndef CURL_SIZEOF_CURL_SOCKLEN_T -# error "CURL_SIZEOF_CURL_SOCKLEN_T definition is missing!" - Error Compilation_aborted_CURL_SIZEOF_CURL_SOCKLEN_T_is_missing -#endif - -#ifndef CURL_TYPEOF_CURL_OFF_T -# error "CURL_TYPEOF_CURL_OFF_T definition is missing!" - Error Compilation_aborted_CURL_TYPEOF_CURL_OFF_T_is_missing -#endif - -#ifndef CURL_FORMAT_CURL_OFF_T -# error "CURL_FORMAT_CURL_OFF_T definition is missing!" - Error Compilation_aborted_CURL_FORMAT_CURL_OFF_T_is_missing -#endif - -#ifndef CURL_FORMAT_CURL_OFF_TU -# error "CURL_FORMAT_CURL_OFF_TU definition is missing!" - Error Compilation_aborted_CURL_FORMAT_CURL_OFF_TU_is_missing -#endif - -#ifndef CURL_FORMAT_OFF_T -# error "CURL_FORMAT_OFF_T definition is missing!" - Error Compilation_aborted_CURL_FORMAT_OFF_T_is_missing -#endif - -#ifndef CURL_SIZEOF_CURL_OFF_T -# error "CURL_SIZEOF_CURL_OFF_T definition is missing!" - Error Compilation_aborted_CURL_SIZEOF_CURL_OFF_T_is_missing -#endif - -#ifndef CURL_SUFFIX_CURL_OFF_T -# error "CURL_SUFFIX_CURL_OFF_T definition is missing!" - Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_T_is_missing -#endif - -#ifndef CURL_SUFFIX_CURL_OFF_TU -# error "CURL_SUFFIX_CURL_OFF_TU definition is missing!" - Error Compilation_aborted_CURL_SUFFIX_CURL_OFF_TU_is_missing -#endif - -/* - * Macros private to this header file. - */ - -#define CurlchkszEQ(t, s) sizeof(t) == s ? 1 : -1 - -#define CurlchkszGE(t1, t2) sizeof(t1) >= sizeof(t2) ? 1 : -1 - -/* - * Verify that the size previously defined and expected for long - * is the same as the one reported by sizeof() at compile time. - */ - -typedef char - __curl_rule_01__ - [CurlchkszEQ(long, CURL_SIZEOF_LONG)]; - -/* - * Verify that the size previously defined and expected for - * curl_off_t is actually the the same as the one reported - * by sizeof() at compile time. - */ - -typedef char - __curl_rule_02__ - [CurlchkszEQ(curl_off_t, CURL_SIZEOF_CURL_OFF_T)]; - -/* - * Verify at compile time that the size of curl_off_t as reported - * by sizeof() is greater or equal than the one reported for long - * for the current compilation. - */ - -typedef char - __curl_rule_03__ - [CurlchkszGE(curl_off_t, long)]; - -/* - * Verify that the size previously defined and expected for - * curl_socklen_t is actually the the same as the one reported - * by sizeof() at compile time. - */ - -typedef char - __curl_rule_04__ - [CurlchkszEQ(curl_socklen_t, CURL_SIZEOF_CURL_SOCKLEN_T)]; - -/* - * Verify at compile time that the size of curl_socklen_t as reported - * by sizeof() is greater or equal than the one reported for int for - * the current compilation. - */ - -typedef char - __curl_rule_05__ - [CurlchkszGE(curl_socklen_t, int)]; - -/* ================================================================ */ -/* EXTERNALLY AND INTERNALLY VISIBLE DEFINITIONS */ -/* ================================================================ */ - -/* - * CURL_ISOCPP and CURL_OFF_T_C definitions are done here in order to allow - * these to be visible and exported by the external libcurl interface API, - * while also making them visible to the library internals, simply including - * curl_setup.h, without actually needing to include curl.h internally. - * If some day this section would grow big enough, all this should be moved - * to its own header file. - */ - -/* - * Figure out if we can use the ## preprocessor operator, which is supported - * by ISO/ANSI C and C++. Some compilers support it without setting __STDC__ - * or __cplusplus so we need to carefully check for them too. - */ - -#if defined(__STDC__) || defined(_MSC_VER) || defined(__cplusplus) || \ - defined(__HP_aCC) || defined(__BORLANDC__) || defined(__LCC__) || \ - defined(__POCC__) || defined(__SALFORDC__) || defined(__HIGHC__) || \ - defined(__ILEC400__) - /* This compiler is believed to have an ISO compatible preprocessor */ -#define CURL_ISOCPP -#else - /* This compiler is believed NOT to have an ISO compatible preprocessor */ -#undef CURL_ISOCPP -#endif - -/* - * Macros for minimum-width signed and unsigned curl_off_t integer constants. - */ - -#if defined(__BORLANDC__) && (__BORLANDC__ == 0x0551) -# define __CURL_OFF_T_C_HLPR2(x) x -# define __CURL_OFF_T_C_HLPR1(x) __CURL_OFF_T_C_HLPR2(x) -# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ - __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_T) -# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val) ## \ - __CURL_OFF_T_C_HLPR1(CURL_SUFFIX_CURL_OFF_TU) -#else -# ifdef CURL_ISOCPP -# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val ## Suffix -# else -# define __CURL_OFF_T_C_HLPR2(Val,Suffix) Val/**/Suffix -# endif -# define __CURL_OFF_T_C_HLPR1(Val,Suffix) __CURL_OFF_T_C_HLPR2(Val,Suffix) -# define CURL_OFF_T_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_T) -# define CURL_OFF_TU_C(Val) __CURL_OFF_T_C_HLPR1(Val,CURL_SUFFIX_CURL_OFF_TU) -#endif - -/* - * Get rid of macros private to this header file. - */ - -#undef CurlchkszEQ -#undef CurlchkszGE - -/* - * Get rid of macros not intended to exist beyond this point. - */ - -#undef CURL_PULL_WS2TCPIP_H -#undef CURL_PULL_SYS_TYPES_H -#undef CURL_PULL_SYS_SOCKET_H -#undef CURL_PULL_SYS_POLL_H -#undef CURL_PULL_STDINT_H -#undef CURL_PULL_INTTYPES_H - -#undef CURL_TYPEOF_CURL_SOCKLEN_T -#undef CURL_TYPEOF_CURL_OFF_T - -#ifdef CURL_NO_OLDIES -#undef CURL_FORMAT_OFF_T /* not required since 7.19.0 - obsoleted in 7.20.0 */ -#endif - -#endif /* __CURL_CURLRULES_H */ diff --git a/deps/libcurl/include/curl/curlver.h b/deps/libcurl/include/curl/curlver.h deleted file mode 100644 index 17906764c898e69ce28204bda7d74573f4c365ee..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/curlver.h +++ /dev/null @@ -1,77 +0,0 @@ -#ifndef __CURL_CURLVER_H -#define __CURL_CURLVER_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ - -/* This header file contains nothing but libcurl version info, generated by - a script at release-time. This was made its own header file in 7.11.2 */ - -/* This is the global package copyright */ -#define LIBCURL_COPYRIGHT "1996 - 2015 Daniel Stenberg, ." - -/* This is the version number of the libcurl package from which this header - file origins: */ -#define LIBCURL_VERSION "7.47.0" - -/* The numeric version number is also available "in parts" by using these - defines: */ -#define LIBCURL_VERSION_MAJOR 7 -#define LIBCURL_VERSION_MINOR 47 -#define LIBCURL_VERSION_PATCH 0 - -/* This is the numeric version of the libcurl version number, meant for easier - parsing and comparions by programs. The LIBCURL_VERSION_NUM define will - always follow this syntax: - - 0xXXYYZZ - - Where XX, YY and ZZ are the main version, release and patch numbers in - hexadecimal (using 8 bits each). All three numbers are always represented - using two digits. 1.2 would appear as "0x010200" while version 9.11.7 - appears as "0x090b07". - - This 6-digit (24 bits) hexadecimal number does not show pre-release number, - and it is always a greater number in a more recent release. It makes - comparisons with greater than and less than work. - - Note: This define is the full hex number and _does not_ use the - CURL_VERSION_BITS() macro since curl's own configure script greps for it - and needs it to contain the full number. -*/ -#define LIBCURL_VERSION_NUM 0x072f00 - -/* - * This is the date and time when the full source package was created. The - * timestamp is not stored in git, as the timestamp is properly set in the - * tarballs by the maketgz script. - * - * The format of the date should follow this template: - * - * "Mon Feb 12 11:35:33 UTC 2007" - */ -#define LIBCURL_TIMESTAMP "Wed Jan 27 07:32:44 UTC 2016" - -#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z) -#define CURL_AT_LEAST_VERSION(x,y,z) \ - (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z)) - -#endif /* __CURL_CURLVER_H */ diff --git a/deps/libcurl/include/curl/easy.h b/deps/libcurl/include/curl/easy.h deleted file mode 100644 index c1e3e76096e3925821da279a2f2eae67ae61e0e8..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/easy.h +++ /dev/null @@ -1,102 +0,0 @@ -#ifndef __CURL_EASY_H -#define __CURL_EASY_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2008, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ -#ifdef __cplusplus -extern "C" { -#endif - -CURL_EXTERN CURL *curl_easy_init(void); -CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); -CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); -CURL_EXTERN void curl_easy_cleanup(CURL *curl); - -/* - * NAME curl_easy_getinfo() - * - * DESCRIPTION - * - * Request internal information from the curl session with this function. The - * third argument MUST be a pointer to a long, a pointer to a char * or a - * pointer to a double (as the documentation describes elsewhere). The data - * pointed to will be filled in accordingly and can be relied upon only if the - * function returns CURLE_OK. This function is intended to get used *AFTER* a - * performed transfer, all results from this function are undefined until the - * transfer is completed. - */ -CURL_EXTERN CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...); - - -/* - * NAME curl_easy_duphandle() - * - * DESCRIPTION - * - * Creates a new curl session handle with the same options set for the handle - * passed in. Duplicating a handle could only be a matter of cloning data and - * options, internal state info and things like persistent connections cannot - * be transferred. It is useful in multithreaded applications when you can run - * curl_easy_duphandle() for each new thread to avoid a series of identical - * curl_easy_setopt() invokes in every thread. - */ -CURL_EXTERN CURL* curl_easy_duphandle(CURL *curl); - -/* - * NAME curl_easy_reset() - * - * DESCRIPTION - * - * Re-initializes a CURL handle to the default values. This puts back the - * handle to the same state as it was in when it was just created. - * - * It does keep: live connections, the Session ID cache, the DNS cache and the - * cookies. - */ -CURL_EXTERN void curl_easy_reset(CURL *curl); - -/* - * NAME curl_easy_recv() - * - * DESCRIPTION - * - * Receives data from the connected socket. Use after successful - * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. - */ -CURL_EXTERN CURLcode curl_easy_recv(CURL *curl, void *buffer, size_t buflen, - size_t *n); - -/* - * NAME curl_easy_send() - * - * DESCRIPTION - * - * Sends data over the connected socket. Use after successful - * curl_easy_perform() with CURLOPT_CONNECT_ONLY option. - */ -CURL_EXTERN CURLcode curl_easy_send(CURL *curl, const void *buffer, - size_t buflen, size_t *n); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/deps/libcurl/include/curl/mprintf.h b/deps/libcurl/include/curl/mprintf.h deleted file mode 100644 index c6b0d7679a6cf970a7960af9f47a7485729598dc..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/mprintf.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef __CURL_MPRINTF_H -#define __CURL_MPRINTF_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ - -#include -#include /* needed for FILE */ - -#include "curl.h" - -#ifdef __cplusplus -extern "C" { -#endif - -CURL_EXTERN int curl_mprintf(const char *format, ...); -CURL_EXTERN int curl_mfprintf(FILE *fd, const char *format, ...); -CURL_EXTERN int curl_msprintf(char *buffer, const char *format, ...); -CURL_EXTERN int curl_msnprintf(char *buffer, size_t maxlength, - const char *format, ...); -CURL_EXTERN int curl_mvprintf(const char *format, va_list args); -CURL_EXTERN int curl_mvfprintf(FILE *fd, const char *format, va_list args); -CURL_EXTERN int curl_mvsprintf(char *buffer, const char *format, va_list args); -CURL_EXTERN int curl_mvsnprintf(char *buffer, size_t maxlength, - const char *format, va_list args); -CURL_EXTERN char *curl_maprintf(const char *format, ...); -CURL_EXTERN char *curl_mvaprintf(const char *format, va_list args); - -#ifdef _MPRINTF_REPLACE -# undef printf -# undef fprintf -# undef sprintf -# undef vsprintf -# undef snprintf -# undef vprintf -# undef vfprintf -# undef vsnprintf -# undef aprintf -# undef vaprintf -# define printf curl_mprintf -# define fprintf curl_mfprintf -# define sprintf curl_msprintf -# define vsprintf curl_mvsprintf -# define snprintf curl_msnprintf -# define vprintf curl_mvprintf -# define vfprintf curl_mvfprintf -# define vsnprintf curl_mvsnprintf -# define aprintf curl_maprintf -# define vaprintf curl_mvaprintf -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* __CURL_MPRINTF_H */ diff --git a/deps/libcurl/include/curl/multi.h b/deps/libcurl/include/curl/multi.h deleted file mode 100644 index 36e2e940ecd440c0a14aadc0dc3eba1d5d53f186..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/multi.h +++ /dev/null @@ -1,435 +0,0 @@ -#ifndef __CURL_MULTI_H -#define __CURL_MULTI_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ -/* - This is an "external" header file. Don't give away any internals here! - - GOALS - - o Enable a "pull" interface. The application that uses libcurl decides where - and when to ask libcurl to get/send data. - - o Enable multiple simultaneous transfers in the same thread without making it - complicated for the application. - - o Enable the application to select() on its own file descriptors and curl's - file descriptors simultaneous easily. - -*/ - -/* - * This header file should not really need to include "curl.h" since curl.h - * itself includes this file and we expect user applications to do #include - * without the need for especially including multi.h. - * - * For some reason we added this include here at one point, and rather than to - * break existing (wrongly written) libcurl applications, we leave it as-is - * but with this warning attached. - */ -#include "curl.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void CURLM; - -typedef enum { - CURLM_CALL_MULTI_PERFORM = -1, /* please call curl_multi_perform() or - curl_multi_socket*() soon */ - CURLM_OK, - CURLM_BAD_HANDLE, /* the passed-in handle is not a valid CURLM handle */ - CURLM_BAD_EASY_HANDLE, /* an easy handle was not good/valid */ - CURLM_OUT_OF_MEMORY, /* if you ever get this, you're in deep sh*t */ - CURLM_INTERNAL_ERROR, /* this is a libcurl bug */ - CURLM_BAD_SOCKET, /* the passed in socket argument did not match */ - CURLM_UNKNOWN_OPTION, /* curl_multi_setopt() with unsupported option */ - CURLM_ADDED_ALREADY, /* an easy handle already added to a multi handle was - attempted to get added - again */ - CURLM_LAST -} CURLMcode; - -/* just to make code nicer when using curl_multi_socket() you can now check - for CURLM_CALL_MULTI_SOCKET too in the same style it works for - curl_multi_perform() and CURLM_CALL_MULTI_PERFORM */ -#define CURLM_CALL_MULTI_SOCKET CURLM_CALL_MULTI_PERFORM - -/* bitmask bits for CURLMOPT_PIPELINING */ -#define CURLPIPE_NOTHING 0L -#define CURLPIPE_HTTP1 1L -#define CURLPIPE_MULTIPLEX 2L - -typedef enum { - CURLMSG_NONE, /* first, not used */ - CURLMSG_DONE, /* This easy handle has completed. 'result' contains - the CURLcode of the transfer */ - CURLMSG_LAST /* last, not used */ -} CURLMSG; - -struct CURLMsg { - CURLMSG msg; /* what this message means */ - CURL *easy_handle; /* the handle it concerns */ - union { - void *whatever; /* message-specific data */ - CURLcode result; /* return code for transfer */ - } data; -}; -typedef struct CURLMsg CURLMsg; - -/* Based on poll(2) structure and values. - * We don't use pollfd and POLL* constants explicitly - * to cover platforms without poll(). */ -#define CURL_WAIT_POLLIN 0x0001 -#define CURL_WAIT_POLLPRI 0x0002 -#define CURL_WAIT_POLLOUT 0x0004 - -struct curl_waitfd { - curl_socket_t fd; - short events; - short revents; /* not supported yet */ -}; - -/* - * Name: curl_multi_init() - * - * Desc: inititalize multi-style curl usage - * - * Returns: a new CURLM handle to use in all 'curl_multi' functions. - */ -CURL_EXTERN CURLM *curl_multi_init(void); - -/* - * Name: curl_multi_add_handle() - * - * Desc: add a standard curl handle to the multi stack - * - * Returns: CURLMcode type, general multi error code. - */ -CURL_EXTERN CURLMcode curl_multi_add_handle(CURLM *multi_handle, - CURL *curl_handle); - - /* - * Name: curl_multi_remove_handle() - * - * Desc: removes a curl handle from the multi stack again - * - * Returns: CURLMcode type, general multi error code. - */ -CURL_EXTERN CURLMcode curl_multi_remove_handle(CURLM *multi_handle, - CURL *curl_handle); - - /* - * Name: curl_multi_fdset() - * - * Desc: Ask curl for its fd_set sets. The app can use these to select() or - * poll() on. We want curl_multi_perform() called as soon as one of - * them are ready. - * - * Returns: CURLMcode type, general multi error code. - */ -CURL_EXTERN CURLMcode curl_multi_fdset(CURLM *multi_handle, - fd_set *read_fd_set, - fd_set *write_fd_set, - fd_set *exc_fd_set, - int *max_fd); - -/* - * Name: curl_multi_wait() - * - * Desc: Poll on all fds within a CURLM set as well as any - * additional fds passed to the function. - * - * Returns: CURLMcode type, general multi error code. - */ -CURL_EXTERN CURLMcode curl_multi_wait(CURLM *multi_handle, - struct curl_waitfd extra_fds[], - unsigned int extra_nfds, - int timeout_ms, - int *ret); - - /* - * Name: curl_multi_perform() - * - * Desc: When the app thinks there's data available for curl it calls this - * function to read/write whatever there is right now. This returns - * as soon as the reads and writes are done. This function does not - * require that there actually is data available for reading or that - * data can be written, it can be called just in case. It returns - * the number of handles that still transfer data in the second - * argument's integer-pointer. - * - * Returns: CURLMcode type, general multi error code. *NOTE* that this only - * returns errors etc regarding the whole multi stack. There might - * still have occurred problems on invidual transfers even when this - * returns OK. - */ -CURL_EXTERN CURLMcode curl_multi_perform(CURLM *multi_handle, - int *running_handles); - - /* - * Name: curl_multi_cleanup() - * - * Desc: Cleans up and removes a whole multi stack. It does not free or - * touch any individual easy handles in any way. We need to define - * in what state those handles will be if this function is called - * in the middle of a transfer. - * - * Returns: CURLMcode type, general multi error code. - */ -CURL_EXTERN CURLMcode curl_multi_cleanup(CURLM *multi_handle); - -/* - * Name: curl_multi_info_read() - * - * Desc: Ask the multi handle if there's any messages/informationals from - * the individual transfers. Messages include informationals such as - * error code from the transfer or just the fact that a transfer is - * completed. More details on these should be written down as well. - * - * Repeated calls to this function will return a new struct each - * time, until a special "end of msgs" struct is returned as a signal - * that there is no more to get at this point. - * - * The data the returned pointer points to will not survive calling - * curl_multi_cleanup(). - * - * The 'CURLMsg' struct is meant to be very simple and only contain - * very basic informations. If more involved information is wanted, - * we will provide the particular "transfer handle" in that struct - * and that should/could/would be used in subsequent - * curl_easy_getinfo() calls (or similar). The point being that we - * must never expose complex structs to applications, as then we'll - * undoubtably get backwards compatibility problems in the future. - * - * Returns: A pointer to a filled-in struct, or NULL if it failed or ran out - * of structs. It also writes the number of messages left in the - * queue (after this read) in the integer the second argument points - * to. - */ -CURL_EXTERN CURLMsg *curl_multi_info_read(CURLM *multi_handle, - int *msgs_in_queue); - -/* - * Name: curl_multi_strerror() - * - * Desc: The curl_multi_strerror function may be used to turn a CURLMcode - * value into the equivalent human readable error string. This is - * useful for printing meaningful error messages. - * - * Returns: A pointer to a zero-terminated error message. - */ -CURL_EXTERN const char *curl_multi_strerror(CURLMcode); - -/* - * Name: curl_multi_socket() and - * curl_multi_socket_all() - * - * Desc: An alternative version of curl_multi_perform() that allows the - * application to pass in one of the file descriptors that have been - * detected to have "action" on them and let libcurl perform. - * See man page for details. - */ -#define CURL_POLL_NONE 0 -#define CURL_POLL_IN 1 -#define CURL_POLL_OUT 2 -#define CURL_POLL_INOUT 3 -#define CURL_POLL_REMOVE 4 - -#define CURL_SOCKET_TIMEOUT CURL_SOCKET_BAD - -#define CURL_CSELECT_IN 0x01 -#define CURL_CSELECT_OUT 0x02 -#define CURL_CSELECT_ERR 0x04 - -typedef int (*curl_socket_callback)(CURL *easy, /* easy handle */ - curl_socket_t s, /* socket */ - int what, /* see above */ - void *userp, /* private callback - pointer */ - void *socketp); /* private socket - pointer */ -/* - * Name: curl_multi_timer_callback - * - * Desc: Called by libcurl whenever the library detects a change in the - * maximum number of milliseconds the app is allowed to wait before - * curl_multi_socket() or curl_multi_perform() must be called - * (to allow libcurl's timed events to take place). - * - * Returns: The callback should return zero. - */ -typedef int (*curl_multi_timer_callback)(CURLM *multi, /* multi handle */ - long timeout_ms, /* see above */ - void *userp); /* private callback - pointer */ - -CURL_EXTERN CURLMcode curl_multi_socket(CURLM *multi_handle, curl_socket_t s, - int *running_handles); - -CURL_EXTERN CURLMcode curl_multi_socket_action(CURLM *multi_handle, - curl_socket_t s, - int ev_bitmask, - int *running_handles); - -CURL_EXTERN CURLMcode curl_multi_socket_all(CURLM *multi_handle, - int *running_handles); - -#ifndef CURL_ALLOW_OLD_MULTI_SOCKET -/* This macro below was added in 7.16.3 to push users who recompile to use - the new curl_multi_socket_action() instead of the old curl_multi_socket() -*/ -#define curl_multi_socket(x,y,z) curl_multi_socket_action(x,y,0,z) -#endif - -/* - * Name: curl_multi_timeout() - * - * Desc: Returns the maximum number of milliseconds the app is allowed to - * wait before curl_multi_socket() or curl_multi_perform() must be - * called (to allow libcurl's timed events to take place). - * - * Returns: CURLM error code. - */ -CURL_EXTERN CURLMcode curl_multi_timeout(CURLM *multi_handle, - long *milliseconds); - -#undef CINIT /* re-using the same name as in curl.h */ - -#ifdef CURL_ISOCPP -#define CINIT(name,type,num) CURLMOPT_ ## name = CURLOPTTYPE_ ## type + num -#else -/* The macro "##" is ISO C, we assume pre-ISO C doesn't support it. */ -#define LONG CURLOPTTYPE_LONG -#define OBJECTPOINT CURLOPTTYPE_OBJECTPOINT -#define FUNCTIONPOINT CURLOPTTYPE_FUNCTIONPOINT -#define OFF_T CURLOPTTYPE_OFF_T -#define CINIT(name,type,number) CURLMOPT_/**/name = type + number -#endif - -typedef enum { - /* This is the socket callback function pointer */ - CINIT(SOCKETFUNCTION, FUNCTIONPOINT, 1), - - /* This is the argument passed to the socket callback */ - CINIT(SOCKETDATA, OBJECTPOINT, 2), - - /* set to 1 to enable pipelining for this multi handle */ - CINIT(PIPELINING, LONG, 3), - - /* This is the timer callback function pointer */ - CINIT(TIMERFUNCTION, FUNCTIONPOINT, 4), - - /* This is the argument passed to the timer callback */ - CINIT(TIMERDATA, OBJECTPOINT, 5), - - /* maximum number of entries in the connection cache */ - CINIT(MAXCONNECTS, LONG, 6), - - /* maximum number of (pipelining) connections to one host */ - CINIT(MAX_HOST_CONNECTIONS, LONG, 7), - - /* maximum number of requests in a pipeline */ - CINIT(MAX_PIPELINE_LENGTH, LONG, 8), - - /* a connection with a content-length longer than this - will not be considered for pipelining */ - CINIT(CONTENT_LENGTH_PENALTY_SIZE, OFF_T, 9), - - /* a connection with a chunk length longer than this - will not be considered for pipelining */ - CINIT(CHUNK_LENGTH_PENALTY_SIZE, OFF_T, 10), - - /* a list of site names(+port) that are blacklisted from - pipelining */ - CINIT(PIPELINING_SITE_BL, OBJECTPOINT, 11), - - /* a list of server types that are blacklisted from - pipelining */ - CINIT(PIPELINING_SERVER_BL, OBJECTPOINT, 12), - - /* maximum number of open connections in total */ - CINIT(MAX_TOTAL_CONNECTIONS, LONG, 13), - - /* This is the server push callback function pointer */ - CINIT(PUSHFUNCTION, FUNCTIONPOINT, 14), - - /* This is the argument passed to the server push callback */ - CINIT(PUSHDATA, OBJECTPOINT, 15), - - CURLMOPT_LASTENTRY /* the last unused */ -} CURLMoption; - - -/* - * Name: curl_multi_setopt() - * - * Desc: Sets options for the multi handle. - * - * Returns: CURLM error code. - */ -CURL_EXTERN CURLMcode curl_multi_setopt(CURLM *multi_handle, - CURLMoption option, ...); - - -/* - * Name: curl_multi_assign() - * - * Desc: This function sets an association in the multi handle between the - * given socket and a private pointer of the application. This is - * (only) useful for curl_multi_socket uses. - * - * Returns: CURLM error code. - */ -CURL_EXTERN CURLMcode curl_multi_assign(CURLM *multi_handle, - curl_socket_t sockfd, void *sockp); - - -/* - * Name: curl_push_callback - * - * Desc: This callback gets called when a new stream is being pushed by the - * server. It approves or denies the new stream. - * - * Returns: CURL_PUSH_OK or CURL_PUSH_DENY. - */ -#define CURL_PUSH_OK 0 -#define CURL_PUSH_DENY 1 - -struct curl_pushheaders; /* forward declaration only */ - -CURL_EXTERN char *curl_pushheader_bynum(struct curl_pushheaders *h, - size_t num); -CURL_EXTERN char *curl_pushheader_byname(struct curl_pushheaders *h, - const char *name); - -typedef int (*curl_push_callback)(CURL *parent, - CURL *easy, - size_t num_headers, - struct curl_pushheaders *headers, - void *userp); - -#ifdef __cplusplus -} /* end of extern "C" */ -#endif - -#endif diff --git a/deps/libcurl/include/curl/stdcheaders.h b/deps/libcurl/include/curl/stdcheaders.h deleted file mode 100644 index ad82ef6335d6167aecf8b9e49d68c462ea028bfd..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/stdcheaders.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef __STDC_HEADERS_H -#define __STDC_HEADERS_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2010, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ - -#include - -size_t fread (void *, size_t, size_t, FILE *); -size_t fwrite (const void *, size_t, size_t, FILE *); - -int strcasecmp(const char *, const char *); -int strncasecmp(const char *, const char *, size_t); - -#endif /* __STDC_HEADERS_H */ diff --git a/deps/libcurl/include/curl/typecheck-gcc.h b/deps/libcurl/include/curl/typecheck-gcc.h deleted file mode 100644 index 2e24db0ff59c40b137ec89a9d8a3f571ed4791b2..0000000000000000000000000000000000000000 --- a/deps/libcurl/include/curl/typecheck-gcc.h +++ /dev/null @@ -1,622 +0,0 @@ -#ifndef __CURL_TYPECHECK_GCC_H -#define __CURL_TYPECHECK_GCC_H -/*************************************************************************** - * _ _ ____ _ - * Project ___| | | | _ \| | - * / __| | | | |_) | | - * | (__| |_| | _ <| |___ - * \___|\___/|_| \_\_____| - * - * Copyright (C) 1998 - 2015, Daniel Stenberg, , et al. - * - * This software is licensed as described in the file COPYING, which - * you should have received as part of this distribution. The terms - * are also available at http://curl.haxx.se/docs/copyright.html. - * - * You may opt to use, copy, modify, merge, publish, distribute and/or sell - * copies of the Software, and permit persons to whom the Software is - * furnished to do so, under the terms of the COPYING file. - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY - * KIND, either express or implied. - * - ***************************************************************************/ - -/* wraps curl_easy_setopt() with typechecking */ - -/* To add a new kind of warning, add an - * if(_curl_is_sometype_option(_curl_opt)) - * if(!_curl_is_sometype(value)) - * _curl_easy_setopt_err_sometype(); - * block and define _curl_is_sometype_option, _curl_is_sometype and - * _curl_easy_setopt_err_sometype below - * - * NOTE: We use two nested 'if' statements here instead of the && operator, in - * order to work around gcc bug #32061. It affects only gcc 4.3.x/4.4.x - * when compiling with -Wlogical-op. - * - * To add an option that uses the same type as an existing option, you'll just - * need to extend the appropriate _curl_*_option macro - */ -#define curl_easy_setopt(handle, option, value) \ -__extension__ ({ \ - __typeof__ (option) _curl_opt = option; \ - if(__builtin_constant_p(_curl_opt)) { \ - if(_curl_is_long_option(_curl_opt)) \ - if(!_curl_is_long(value)) \ - _curl_easy_setopt_err_long(); \ - if(_curl_is_off_t_option(_curl_opt)) \ - if(!_curl_is_off_t(value)) \ - _curl_easy_setopt_err_curl_off_t(); \ - if(_curl_is_string_option(_curl_opt)) \ - if(!_curl_is_string(value)) \ - _curl_easy_setopt_err_string(); \ - if(_curl_is_write_cb_option(_curl_opt)) \ - if(!_curl_is_write_cb(value)) \ - _curl_easy_setopt_err_write_callback(); \ - if((_curl_opt) == CURLOPT_READFUNCTION) \ - if(!_curl_is_read_cb(value)) \ - _curl_easy_setopt_err_read_cb(); \ - if((_curl_opt) == CURLOPT_IOCTLFUNCTION) \ - if(!_curl_is_ioctl_cb(value)) \ - _curl_easy_setopt_err_ioctl_cb(); \ - if((_curl_opt) == CURLOPT_SOCKOPTFUNCTION) \ - if(!_curl_is_sockopt_cb(value)) \ - _curl_easy_setopt_err_sockopt_cb(); \ - if((_curl_opt) == CURLOPT_OPENSOCKETFUNCTION) \ - if(!_curl_is_opensocket_cb(value)) \ - _curl_easy_setopt_err_opensocket_cb(); \ - if((_curl_opt) == CURLOPT_PROGRESSFUNCTION) \ - if(!_curl_is_progress_cb(value)) \ - _curl_easy_setopt_err_progress_cb(); \ - if((_curl_opt) == CURLOPT_DEBUGFUNCTION) \ - if(!_curl_is_debug_cb(value)) \ - _curl_easy_setopt_err_debug_cb(); \ - if((_curl_opt) == CURLOPT_SSL_CTX_FUNCTION) \ - if(!_curl_is_ssl_ctx_cb(value)) \ - _curl_easy_setopt_err_ssl_ctx_cb(); \ - if(_curl_is_conv_cb_option(_curl_opt)) \ - if(!_curl_is_conv_cb(value)) \ - _curl_easy_setopt_err_conv_cb(); \ - if((_curl_opt) == CURLOPT_SEEKFUNCTION) \ - if(!_curl_is_seek_cb(value)) \ - _curl_easy_setopt_err_seek_cb(); \ - if(_curl_is_cb_data_option(_curl_opt)) \ - if(!_curl_is_cb_data(value)) \ - _curl_easy_setopt_err_cb_data(); \ - if((_curl_opt) == CURLOPT_ERRORBUFFER) \ - if(!_curl_is_error_buffer(value)) \ - _curl_easy_setopt_err_error_buffer(); \ - if((_curl_opt) == CURLOPT_STDERR) \ - if(!_curl_is_FILE(value)) \ - _curl_easy_setopt_err_FILE(); \ - if(_curl_is_postfields_option(_curl_opt)) \ - if(!_curl_is_postfields(value)) \ - _curl_easy_setopt_err_postfields(); \ - if((_curl_opt) == CURLOPT_HTTPPOST) \ - if(!_curl_is_arr((value), struct curl_httppost)) \ - _curl_easy_setopt_err_curl_httpost(); \ - if(_curl_is_slist_option(_curl_opt)) \ - if(!_curl_is_arr((value), struct curl_slist)) \ - _curl_easy_setopt_err_curl_slist(); \ - if((_curl_opt) == CURLOPT_SHARE) \ - if(!_curl_is_ptr((value), CURLSH)) \ - _curl_easy_setopt_err_CURLSH(); \ - } \ - curl_easy_setopt(handle, _curl_opt, value); \ -}) - -/* wraps curl_easy_getinfo() with typechecking */ -/* FIXME: don't allow const pointers */ -#define curl_easy_getinfo(handle, info, arg) \ -__extension__ ({ \ - __typeof__ (info) _curl_info = info; \ - if(__builtin_constant_p(_curl_info)) { \ - if(_curl_is_string_info(_curl_info)) \ - if(!_curl_is_arr((arg), char *)) \ - _curl_easy_getinfo_err_string(); \ - if(_curl_is_long_info(_curl_info)) \ - if(!_curl_is_arr((arg), long)) \ - _curl_easy_getinfo_err_long(); \ - if(_curl_is_double_info(_curl_info)) \ - if(!_curl_is_arr((arg), double)) \ - _curl_easy_getinfo_err_double(); \ - if(_curl_is_slist_info(_curl_info)) \ - if(!_curl_is_arr((arg), struct curl_slist *)) \ - _curl_easy_getinfo_err_curl_slist(); \ - } \ - curl_easy_getinfo(handle, _curl_info, arg); \ -}) - -/* TODO: typechecking for curl_share_setopt() and curl_multi_setopt(), - * for now just make sure that the functions are called with three - * arguments - */ -#define curl_share_setopt(share,opt,param) curl_share_setopt(share,opt,param) -#define curl_multi_setopt(handle,opt,param) curl_multi_setopt(handle,opt,param) - - -/* the actual warnings, triggered by calling the _curl_easy_setopt_err* - * functions */ - -/* To define a new warning, use _CURL_WARNING(identifier, "message") */ -#define _CURL_WARNING(id, message) \ - static void __attribute__((__warning__(message))) \ - __attribute__((__unused__)) __attribute__((__noinline__)) \ - id(void) { __asm__(""); } - -_CURL_WARNING(_curl_easy_setopt_err_long, - "curl_easy_setopt expects a long argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_curl_off_t, - "curl_easy_setopt expects a curl_off_t argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_string, - "curl_easy_setopt expects a " - "string (char* or char[]) argument for this option" - ) -_CURL_WARNING(_curl_easy_setopt_err_write_callback, - "curl_easy_setopt expects a curl_write_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_read_cb, - "curl_easy_setopt expects a curl_read_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_ioctl_cb, - "curl_easy_setopt expects a curl_ioctl_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_sockopt_cb, - "curl_easy_setopt expects a curl_sockopt_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_opensocket_cb, - "curl_easy_setopt expects a " - "curl_opensocket_callback argument for this option" - ) -_CURL_WARNING(_curl_easy_setopt_err_progress_cb, - "curl_easy_setopt expects a curl_progress_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_debug_cb, - "curl_easy_setopt expects a curl_debug_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_ssl_ctx_cb, - "curl_easy_setopt expects a curl_ssl_ctx_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_conv_cb, - "curl_easy_setopt expects a curl_conv_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_seek_cb, - "curl_easy_setopt expects a curl_seek_callback argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_cb_data, - "curl_easy_setopt expects a " - "private data pointer as argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_error_buffer, - "curl_easy_setopt expects a " - "char buffer of CURL_ERROR_SIZE as argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_FILE, - "curl_easy_setopt expects a FILE* argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_postfields, - "curl_easy_setopt expects a void* or char* argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_curl_httpost, - "curl_easy_setopt expects a struct curl_httppost* argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_curl_slist, - "curl_easy_setopt expects a struct curl_slist* argument for this option") -_CURL_WARNING(_curl_easy_setopt_err_CURLSH, - "curl_easy_setopt expects a CURLSH* argument for this option") - -_CURL_WARNING(_curl_easy_getinfo_err_string, - "curl_easy_getinfo expects a pointer to char * for this info") -_CURL_WARNING(_curl_easy_getinfo_err_long, - "curl_easy_getinfo expects a pointer to long for this info") -_CURL_WARNING(_curl_easy_getinfo_err_double, - "curl_easy_getinfo expects a pointer to double for this info") -_CURL_WARNING(_curl_easy_getinfo_err_curl_slist, - "curl_easy_getinfo expects a pointer to struct curl_slist * for this info") - -/* groups of curl_easy_setops options that take the same type of argument */ - -/* To add a new option to one of the groups, just add - * (option) == CURLOPT_SOMETHING - * to the or-expression. If the option takes a long or curl_off_t, you don't - * have to do anything - */ - -/* evaluates to true if option takes a long argument */ -#define _curl_is_long_option(option) \ - (0 < (option) && (option) < CURLOPTTYPE_OBJECTPOINT) - -#define _curl_is_off_t_option(option) \ - ((option) > CURLOPTTYPE_OFF_T) - -/* evaluates to true if option takes a char* argument */ -#define _curl_is_string_option(option) \ - ((option) == CURLOPT_ACCEPT_ENCODING || \ - (option) == CURLOPT_CAINFO || \ - (option) == CURLOPT_CAPATH || \ - (option) == CURLOPT_COOKIE || \ - (option) == CURLOPT_COOKIEFILE || \ - (option) == CURLOPT_COOKIEJAR || \ - (option) == CURLOPT_COOKIELIST || \ - (option) == CURLOPT_CRLFILE || \ - (option) == CURLOPT_CUSTOMREQUEST || \ - (option) == CURLOPT_DEFAULT_PROTOCOL || \ - (option) == CURLOPT_DNS_INTERFACE || \ - (option) == CURLOPT_DNS_LOCAL_IP4 || \ - (option) == CURLOPT_DNS_LOCAL_IP6 || \ - (option) == CURLOPT_DNS_SERVERS || \ - (option) == CURLOPT_EGDSOCKET || \ - (option) == CURLOPT_FTPPORT || \ - (option) == CURLOPT_FTP_ACCOUNT || \ - (option) == CURLOPT_FTP_ALTERNATIVE_TO_USER || \ - (option) == CURLOPT_INTERFACE || \ - (option) == CURLOPT_ISSUERCERT || \ - (option) == CURLOPT_KEYPASSWD || \ - (option) == CURLOPT_KRBLEVEL || \ - (option) == CURLOPT_LOGIN_OPTIONS || \ - (option) == CURLOPT_MAIL_AUTH || \ - (option) == CURLOPT_MAIL_FROM || \ - (option) == CURLOPT_NETRC_FILE || \ - (option) == CURLOPT_NOPROXY || \ - (option) == CURLOPT_PASSWORD || \ - (option) == CURLOPT_PINNEDPUBLICKEY || \ - (option) == CURLOPT_PROXY || \ - (option) == CURLOPT_PROXYPASSWORD || \ - (option) == CURLOPT_PROXYUSERNAME || \ - (option) == CURLOPT_PROXYUSERPWD || \ - (option) == CURLOPT_PROXY_SERVICE_NAME || \ - (option) == CURLOPT_RANDOM_FILE || \ - (option) == CURLOPT_RANGE || \ - (option) == CURLOPT_REFERER || \ - (option) == CURLOPT_RTSP_SESSION_ID || \ - (option) == CURLOPT_RTSP_STREAM_URI || \ - (option) == CURLOPT_RTSP_TRANSPORT || \ - (option) == CURLOPT_SERVICE_NAME || \ - (option) == CURLOPT_SOCKS5_GSSAPI_SERVICE || \ - (option) == CURLOPT_SSH_HOST_PUBLIC_KEY_MD5 || \ - (option) == CURLOPT_SSH_KNOWNHOSTS || \ - (option) == CURLOPT_SSH_PRIVATE_KEYFILE || \ - (option) == CURLOPT_SSH_PUBLIC_KEYFILE || \ - (option) == CURLOPT_SSLCERT || \ - (option) == CURLOPT_SSLCERTTYPE || \ - (option) == CURLOPT_SSLENGINE || \ - (option) == CURLOPT_SSLKEY || \ - (option) == CURLOPT_SSLKEYTYPE || \ - (option) == CURLOPT_SSL_CIPHER_LIST || \ - (option) == CURLOPT_TLSAUTH_PASSWORD || \ - (option) == CURLOPT_TLSAUTH_TYPE || \ - (option) == CURLOPT_TLSAUTH_USERNAME || \ - (option) == CURLOPT_UNIX_SOCKET_PATH || \ - (option) == CURLOPT_URL || \ - (option) == CURLOPT_USERAGENT || \ - (option) == CURLOPT_USERNAME || \ - (option) == CURLOPT_USERPWD || \ - (option) == CURLOPT_XOAUTH2_BEARER || \ - 0) - -/* evaluates to true if option takes a curl_write_callback argument */ -#define _curl_is_write_cb_option(option) \ - ((option) == CURLOPT_HEADERFUNCTION || \ - (option) == CURLOPT_WRITEFUNCTION) - -/* evaluates to true if option takes a curl_conv_callback argument */ -#define _curl_is_conv_cb_option(option) \ - ((option) == CURLOPT_CONV_TO_NETWORK_FUNCTION || \ - (option) == CURLOPT_CONV_FROM_NETWORK_FUNCTION || \ - (option) == CURLOPT_CONV_FROM_UTF8_FUNCTION) - -/* evaluates to true if option takes a data argument to pass to a callback */ -#define _curl_is_cb_data_option(option) \ - ((option) == CURLOPT_CHUNK_DATA || \ - (option) == CURLOPT_CLOSESOCKETDATA || \ - (option) == CURLOPT_DEBUGDATA || \ - (option) == CURLOPT_FNMATCH_DATA || \ - (option) == CURLOPT_HEADERDATA || \ - (option) == CURLOPT_INTERLEAVEDATA || \ - (option) == CURLOPT_IOCTLDATA || \ - (option) == CURLOPT_OPENSOCKETDATA || \ - (option) == CURLOPT_PRIVATE || \ - (option) == CURLOPT_PROGRESSDATA || \ - (option) == CURLOPT_READDATA || \ - (option) == CURLOPT_SEEKDATA || \ - (option) == CURLOPT_SOCKOPTDATA || \ - (option) == CURLOPT_SSH_KEYDATA || \ - (option) == CURLOPT_SSL_CTX_DATA || \ - (option) == CURLOPT_WRITEDATA || \ - 0) - -/* evaluates to true if option takes a POST data argument (void* or char*) */ -#define _curl_is_postfields_option(option) \ - ((option) == CURLOPT_POSTFIELDS || \ - (option) == CURLOPT_COPYPOSTFIELDS || \ - 0) - -/* evaluates to true if option takes a struct curl_slist * argument */ -#define _curl_is_slist_option(option) \ - ((option) == CURLOPT_HTTP200ALIASES || \ - (option) == CURLOPT_HTTPHEADER || \ - (option) == CURLOPT_MAIL_RCPT || \ - (option) == CURLOPT_POSTQUOTE || \ - (option) == CURLOPT_PREQUOTE || \ - (option) == CURLOPT_PROXYHEADER || \ - (option) == CURLOPT_QUOTE || \ - (option) == CURLOPT_RESOLVE || \ - (option) == CURLOPT_TELNETOPTIONS || \ - 0) - -/* groups of curl_easy_getinfo infos that take the same type of argument */ - -/* evaluates to true if info expects a pointer to char * argument */ -#define _curl_is_string_info(info) \ - (CURLINFO_STRING < (info) && (info) < CURLINFO_LONG) - -/* evaluates to true if info expects a pointer to long argument */ -#define _curl_is_long_info(info) \ - (CURLINFO_LONG < (info) && (info) < CURLINFO_DOUBLE) - -/* evaluates to true if info expects a pointer to double argument */ -#define _curl_is_double_info(info) \ - (CURLINFO_DOUBLE < (info) && (info) < CURLINFO_SLIST) - -/* true if info expects a pointer to struct curl_slist * argument */ -#define _curl_is_slist_info(info) \ - (CURLINFO_SLIST < (info)) - - -/* typecheck helpers -- check whether given expression has requested type*/ - -/* For pointers, you can use the _curl_is_ptr/_curl_is_arr macros, - * otherwise define a new macro. Search for __builtin_types_compatible_p - * in the GCC manual. - * NOTE: these macros MUST NOT EVALUATE their arguments! The argument is - * the actual expression passed to the curl_easy_setopt macro. This - * means that you can only apply the sizeof and __typeof__ operators, no - * == or whatsoever. - */ - -/* XXX: should evaluate to true iff expr is a pointer */ -#define _curl_is_any_ptr(expr) \ - (sizeof(expr) == sizeof(void*)) - -/* evaluates to true if expr is NULL */ -/* XXX: must not evaluate expr, so this check is not accurate */ -#define _curl_is_NULL(expr) \ - (__builtin_types_compatible_p(__typeof__(expr), __typeof__(NULL))) - -/* evaluates to true if expr is type*, const type* or NULL */ -#define _curl_is_ptr(expr, type) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), type *) || \ - __builtin_types_compatible_p(__typeof__(expr), const type *)) - -/* evaluates to true if expr is one of type[], type*, NULL or const type* */ -#define _curl_is_arr(expr, type) \ - (_curl_is_ptr((expr), type) || \ - __builtin_types_compatible_p(__typeof__(expr), type [])) - -/* evaluates to true if expr is a string */ -#define _curl_is_string(expr) \ - (_curl_is_arr((expr), char) || \ - _curl_is_arr((expr), signed char) || \ - _curl_is_arr((expr), unsigned char)) - -/* evaluates to true if expr is a long (no matter the signedness) - * XXX: for now, int is also accepted (and therefore short and char, which - * are promoted to int when passed to a variadic function) */ -#define _curl_is_long(expr) \ - (__builtin_types_compatible_p(__typeof__(expr), long) || \ - __builtin_types_compatible_p(__typeof__(expr), signed long) || \ - __builtin_types_compatible_p(__typeof__(expr), unsigned long) || \ - __builtin_types_compatible_p(__typeof__(expr), int) || \ - __builtin_types_compatible_p(__typeof__(expr), signed int) || \ - __builtin_types_compatible_p(__typeof__(expr), unsigned int) || \ - __builtin_types_compatible_p(__typeof__(expr), short) || \ - __builtin_types_compatible_p(__typeof__(expr), signed short) || \ - __builtin_types_compatible_p(__typeof__(expr), unsigned short) || \ - __builtin_types_compatible_p(__typeof__(expr), char) || \ - __builtin_types_compatible_p(__typeof__(expr), signed char) || \ - __builtin_types_compatible_p(__typeof__(expr), unsigned char)) - -/* evaluates to true if expr is of type curl_off_t */ -#define _curl_is_off_t(expr) \ - (__builtin_types_compatible_p(__typeof__(expr), curl_off_t)) - -/* evaluates to true if expr is abuffer suitable for CURLOPT_ERRORBUFFER */ -/* XXX: also check size of an char[] array? */ -#define _curl_is_error_buffer(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), char *) || \ - __builtin_types_compatible_p(__typeof__(expr), char[])) - -/* evaluates to true if expr is of type (const) void* or (const) FILE* */ -#if 0 -#define _curl_is_cb_data(expr) \ - (_curl_is_ptr((expr), void) || \ - _curl_is_ptr((expr), FILE)) -#else /* be less strict */ -#define _curl_is_cb_data(expr) \ - _curl_is_any_ptr(expr) -#endif - -/* evaluates to true if expr is of type FILE* */ -#define _curl_is_FILE(expr) \ - (__builtin_types_compatible_p(__typeof__(expr), FILE *)) - -/* evaluates to true if expr can be passed as POST data (void* or char*) */ -#define _curl_is_postfields(expr) \ - (_curl_is_ptr((expr), void) || \ - _curl_is_arr((expr), char)) - -/* FIXME: the whole callback checking is messy... - * The idea is to tolerate char vs. void and const vs. not const - * pointers in arguments at least - */ -/* helper: __builtin_types_compatible_p distinguishes between functions and - * function pointers, hide it */ -#define _curl_callback_compatible(func, type) \ - (__builtin_types_compatible_p(__typeof__(func), type) || \ - __builtin_types_compatible_p(__typeof__(func), type*)) - -/* evaluates to true if expr is of type curl_read_callback or "similar" */ -#define _curl_is_read_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), __typeof__(fread)) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_read_callback) || \ - _curl_callback_compatible((expr), _curl_read_callback1) || \ - _curl_callback_compatible((expr), _curl_read_callback2) || \ - _curl_callback_compatible((expr), _curl_read_callback3) || \ - _curl_callback_compatible((expr), _curl_read_callback4) || \ - _curl_callback_compatible((expr), _curl_read_callback5) || \ - _curl_callback_compatible((expr), _curl_read_callback6)) -typedef size_t (_curl_read_callback1)(char *, size_t, size_t, void*); -typedef size_t (_curl_read_callback2)(char *, size_t, size_t, const void*); -typedef size_t (_curl_read_callback3)(char *, size_t, size_t, FILE*); -typedef size_t (_curl_read_callback4)(void *, size_t, size_t, void*); -typedef size_t (_curl_read_callback5)(void *, size_t, size_t, const void*); -typedef size_t (_curl_read_callback6)(void *, size_t, size_t, FILE*); - -/* evaluates to true if expr is of type curl_write_callback or "similar" */ -#define _curl_is_write_cb(expr) \ - (_curl_is_read_cb(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), __typeof__(fwrite)) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_write_callback) || \ - _curl_callback_compatible((expr), _curl_write_callback1) || \ - _curl_callback_compatible((expr), _curl_write_callback2) || \ - _curl_callback_compatible((expr), _curl_write_callback3) || \ - _curl_callback_compatible((expr), _curl_write_callback4) || \ - _curl_callback_compatible((expr), _curl_write_callback5) || \ - _curl_callback_compatible((expr), _curl_write_callback6)) -typedef size_t (_curl_write_callback1)(const char *, size_t, size_t, void*); -typedef size_t (_curl_write_callback2)(const char *, size_t, size_t, - const void*); -typedef size_t (_curl_write_callback3)(const char *, size_t, size_t, FILE*); -typedef size_t (_curl_write_callback4)(const void *, size_t, size_t, void*); -typedef size_t (_curl_write_callback5)(const void *, size_t, size_t, - const void*); -typedef size_t (_curl_write_callback6)(const void *, size_t, size_t, FILE*); - -/* evaluates to true if expr is of type curl_ioctl_callback or "similar" */ -#define _curl_is_ioctl_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_ioctl_callback) || \ - _curl_callback_compatible((expr), _curl_ioctl_callback1) || \ - _curl_callback_compatible((expr), _curl_ioctl_callback2) || \ - _curl_callback_compatible((expr), _curl_ioctl_callback3) || \ - _curl_callback_compatible((expr), _curl_ioctl_callback4)) -typedef curlioerr (_curl_ioctl_callback1)(CURL *, int, void*); -typedef curlioerr (_curl_ioctl_callback2)(CURL *, int, const void*); -typedef curlioerr (_curl_ioctl_callback3)(CURL *, curliocmd, void*); -typedef curlioerr (_curl_ioctl_callback4)(CURL *, curliocmd, const void*); - -/* evaluates to true if expr is of type curl_sockopt_callback or "similar" */ -#define _curl_is_sockopt_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_sockopt_callback) || \ - _curl_callback_compatible((expr), _curl_sockopt_callback1) || \ - _curl_callback_compatible((expr), _curl_sockopt_callback2)) -typedef int (_curl_sockopt_callback1)(void *, curl_socket_t, curlsocktype); -typedef int (_curl_sockopt_callback2)(const void *, curl_socket_t, - curlsocktype); - -/* evaluates to true if expr is of type curl_opensocket_callback or - "similar" */ -#define _curl_is_opensocket_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_opensocket_callback) ||\ - _curl_callback_compatible((expr), _curl_opensocket_callback1) || \ - _curl_callback_compatible((expr), _curl_opensocket_callback2) || \ - _curl_callback_compatible((expr), _curl_opensocket_callback3) || \ - _curl_callback_compatible((expr), _curl_opensocket_callback4)) -typedef curl_socket_t (_curl_opensocket_callback1) - (void *, curlsocktype, struct curl_sockaddr *); -typedef curl_socket_t (_curl_opensocket_callback2) - (void *, curlsocktype, const struct curl_sockaddr *); -typedef curl_socket_t (_curl_opensocket_callback3) - (const void *, curlsocktype, struct curl_sockaddr *); -typedef curl_socket_t (_curl_opensocket_callback4) - (const void *, curlsocktype, const struct curl_sockaddr *); - -/* evaluates to true if expr is of type curl_progress_callback or "similar" */ -#define _curl_is_progress_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_progress_callback) || \ - _curl_callback_compatible((expr), _curl_progress_callback1) || \ - _curl_callback_compatible((expr), _curl_progress_callback2)) -typedef int (_curl_progress_callback1)(void *, - double, double, double, double); -typedef int (_curl_progress_callback2)(const void *, - double, double, double, double); - -/* evaluates to true if expr is of type curl_debug_callback or "similar" */ -#define _curl_is_debug_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_debug_callback) || \ - _curl_callback_compatible((expr), _curl_debug_callback1) || \ - _curl_callback_compatible((expr), _curl_debug_callback2) || \ - _curl_callback_compatible((expr), _curl_debug_callback3) || \ - _curl_callback_compatible((expr), _curl_debug_callback4) || \ - _curl_callback_compatible((expr), _curl_debug_callback5) || \ - _curl_callback_compatible((expr), _curl_debug_callback6) || \ - _curl_callback_compatible((expr), _curl_debug_callback7) || \ - _curl_callback_compatible((expr), _curl_debug_callback8)) -typedef int (_curl_debug_callback1) (CURL *, - curl_infotype, char *, size_t, void *); -typedef int (_curl_debug_callback2) (CURL *, - curl_infotype, char *, size_t, const void *); -typedef int (_curl_debug_callback3) (CURL *, - curl_infotype, const char *, size_t, void *); -typedef int (_curl_debug_callback4) (CURL *, - curl_infotype, const char *, size_t, const void *); -typedef int (_curl_debug_callback5) (CURL *, - curl_infotype, unsigned char *, size_t, void *); -typedef int (_curl_debug_callback6) (CURL *, - curl_infotype, unsigned char *, size_t, const void *); -typedef int (_curl_debug_callback7) (CURL *, - curl_infotype, const unsigned char *, size_t, void *); -typedef int (_curl_debug_callback8) (CURL *, - curl_infotype, const unsigned char *, size_t, const void *); - -/* evaluates to true if expr is of type curl_ssl_ctx_callback or "similar" */ -/* this is getting even messier... */ -#define _curl_is_ssl_ctx_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_ssl_ctx_callback) || \ - _curl_callback_compatible((expr), _curl_ssl_ctx_callback1) || \ - _curl_callback_compatible((expr), _curl_ssl_ctx_callback2) || \ - _curl_callback_compatible((expr), _curl_ssl_ctx_callback3) || \ - _curl_callback_compatible((expr), _curl_ssl_ctx_callback4) || \ - _curl_callback_compatible((expr), _curl_ssl_ctx_callback5) || \ - _curl_callback_compatible((expr), _curl_ssl_ctx_callback6) || \ - _curl_callback_compatible((expr), _curl_ssl_ctx_callback7) || \ - _curl_callback_compatible((expr), _curl_ssl_ctx_callback8)) -typedef CURLcode (_curl_ssl_ctx_callback1)(CURL *, void *, void *); -typedef CURLcode (_curl_ssl_ctx_callback2)(CURL *, void *, const void *); -typedef CURLcode (_curl_ssl_ctx_callback3)(CURL *, const void *, void *); -typedef CURLcode (_curl_ssl_ctx_callback4)(CURL *, const void *, const void *); -#ifdef HEADER_SSL_H -/* hack: if we included OpenSSL's ssl.h, we know about SSL_CTX - * this will of course break if we're included before OpenSSL headers... - */ -typedef CURLcode (_curl_ssl_ctx_callback5)(CURL *, SSL_CTX, void *); -typedef CURLcode (_curl_ssl_ctx_callback6)(CURL *, SSL_CTX, const void *); -typedef CURLcode (_curl_ssl_ctx_callback7)(CURL *, const SSL_CTX, void *); -typedef CURLcode (_curl_ssl_ctx_callback8)(CURL *, const SSL_CTX, - const void *); -#else -typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback5; -typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback6; -typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback7; -typedef _curl_ssl_ctx_callback1 _curl_ssl_ctx_callback8; -#endif - -/* evaluates to true if expr is of type curl_conv_callback or "similar" */ -#define _curl_is_conv_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_conv_callback) || \ - _curl_callback_compatible((expr), _curl_conv_callback1) || \ - _curl_callback_compatible((expr), _curl_conv_callback2) || \ - _curl_callback_compatible((expr), _curl_conv_callback3) || \ - _curl_callback_compatible((expr), _curl_conv_callback4)) -typedef CURLcode (*_curl_conv_callback1)(char *, size_t length); -typedef CURLcode (*_curl_conv_callback2)(const char *, size_t length); -typedef CURLcode (*_curl_conv_callback3)(void *, size_t length); -typedef CURLcode (*_curl_conv_callback4)(const void *, size_t length); - -/* evaluates to true if expr is of type curl_seek_callback or "similar" */ -#define _curl_is_seek_cb(expr) \ - (_curl_is_NULL(expr) || \ - __builtin_types_compatible_p(__typeof__(expr), curl_seek_callback) || \ - _curl_callback_compatible((expr), _curl_seek_callback1) || \ - _curl_callback_compatible((expr), _curl_seek_callback2)) -typedef CURLcode (*_curl_seek_callback1)(void *, curl_off_t, int); -typedef CURLcode (*_curl_seek_callback2)(const void *, curl_off_t, int); - - -#endif /* __CURL_TYPECHECK_GCC_H */ diff --git a/deps/libcurl/lib/libcurl.a b/deps/libcurl/lib/libcurl.a deleted file mode 100644 index c63ac70946dddefb5b3b5c55cf82d693b731cad6..0000000000000000000000000000000000000000 Binary files a/deps/libcurl/lib/libcurl.a and /dev/null differ diff --git a/deps/pthread/CMakeLists.txt b/deps/pthread/CMakeLists.txt index dcd9ed035814ac9efd66340b58b287ccbabe61df..04e5be7472a9b8cbdb384348697b919bf2dd0ece 100644 --- a/deps/pthread/CMakeLists.txt +++ b/deps/pthread/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/regex/CMakeLists.txt b/deps/regex/CMakeLists.txt index c7e983b992fe8a2dd21b8420e652358976663a37..054b093d07c386d7ff9b0ffc4c05909d79b33129 100644 --- a/deps/regex/CMakeLists.txt +++ b/deps/regex/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/rmonotonic/CMakeLists.txt b/deps/rmonotonic/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b870b7d6d7a3817c38bfd4c488f75890fd115b7f --- /dev/null +++ b/deps/rmonotonic/CMakeLists.txt @@ -0,0 +1,11 @@ +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SOURCE_LIST) + +add_definitions(-DUSE_PROCESSOR_CLOCK) + +INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../MsvcLibX/include) + +ADD_LIBRARY(rmonotonic ${SOURCE_LIST}) +TARGET_INCLUDE_DIRECTORIES(rmonotonic PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/inc) +IF (TD_WINDOWS) + TARGET_LINK_LIBRARIES(rmonotonic MsvcLibXw) +ENDIF () diff --git a/deps/rmonotonic/inc/monotonic.h b/deps/rmonotonic/inc/monotonic.h new file mode 100644 index 0000000000000000000000000000000000000000..0850548d36629b0b1bfba04e818937e275cac5bf --- /dev/null +++ b/deps/rmonotonic/inc/monotonic.h @@ -0,0 +1,56 @@ +#ifndef __MONOTONIC_H +#define __MONOTONIC_H +/* The monotonic clock is an always increasing clock source. It is unrelated to + * the actual time of day and should only be used for relative timings. The + * monotonic clock is also not guaranteed to be chronologically precise; there + * may be slight skew/shift from a precise clock. + * + * Depending on system architecture, the monotonic time may be able to be + * retrieved much faster than a normal clock source by using an instruction + * counter on the CPU. On x86 architectures (for example), the RDTSC + * instruction is a very fast clock source for this purpose. + */ + +//#include "fmacros.h" +#include +//#include + +#if defined(_WIN32) || defined(_WIN64) + #define inline +#endif + +/* A counter in micro-seconds. The 'monotime' type is provided for variables + * holding a monotonic time. This will help distinguish & document that the + * variable is associated with the monotonic clock and should not be confused + * with other types of time.*/ +typedef uint64_t monotime; + +/* Retrieve counter of micro-seconds relative to an arbitrary point in time. */ +extern monotime (*getMonotonicUs)(void); + + +/* Call once at startup to initialize the monotonic clock. Though this only + * needs to be called once, it may be called additional times without impact. + * Returns a printable string indicating the type of clock initialized. + * (The returned string is static and doesn't need to be freed.) */ +const char * monotonicInit(); + + +/* Functions to measure elapsed time. Example: + * monotime myTimer; + * elapsedStart(&myTimer); + * while (elapsedMs(myTimer) < 10) {} // loops for 10ms + */ +static inline void elapsedStart(monotime *start_time) { + *start_time = getMonotonicUs(); +} + +static inline uint64_t elapsedUs(monotime start_time) { + return getMonotonicUs() - start_time; +} + +static inline uint64_t elapsedMs(monotime start_time) { + return elapsedUs(start_time) / 1000; +} + +#endif diff --git a/deps/rmonotonic/src/monotonic.c b/deps/rmonotonic/src/monotonic.c new file mode 100644 index 0000000000000000000000000000000000000000..1470f91b56c79b4ee2d8429ecf58fc365d03e737 --- /dev/null +++ b/deps/rmonotonic/src/monotonic.c @@ -0,0 +1,173 @@ +#include "monotonic.h" +#include +#include +#include +#include + +#undef NDEBUG +#include + +#if defined(_WIN32) || defined(_WIN64) +#include "msvcTime.h" +#include "msvcStdio.h" +#endif + +/* The function pointer for clock retrieval. */ +monotime (*getMonotonicUs)(void) = NULL; + +static char monotonic_info_string[32]; + + +/* Using the processor clock (aka TSC on x86) can provide improved performance + * throughout Redis wherever the monotonic clock is used. The processor clock + * is significantly faster than calling 'clock_getting' (POSIX). While this is + * generally safe on modern systems, this link provides additional information + * about use of the x86 TSC: http://oliveryang.net/2015/09/pitfalls-of-TSC-usage + * + * To use the processor clock, either uncomment this line, or build with + * CFLAGS="-DUSE_PROCESSOR_CLOCK" +#define USE_PROCESSOR_CLOCK + */ + + +#if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__) +#include +#include + +static long mono_ticksPerMicrosecond = 0; + +static monotime getMonotonicUs_x86() { + return __rdtsc() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_x86linux() { + const int bufflen = 256; + char buf[bufflen]; + regex_t cpuGhzRegex, constTscRegex; + const size_t nmatch = 2; + regmatch_t pmatch[nmatch]; + int constantTsc = 0; + int rc; + + /* Determine the number of TSC ticks in a micro-second. This is + * a constant value matching the standard speed of the processor. + * On modern processors, this speed remains constant even though + * the actual clock speed varies dynamically for each core. */ + rc = regcomp(&cpuGhzRegex, "^model name\\s+:.*@ ([0-9.]+)GHz", REG_EXTENDED); + assert(rc == 0); + + /* Also check that the constant_tsc flag is present. (It should be + * unless this is a really old CPU. */ + rc = regcomp(&constTscRegex, "^flags\\s+:.* constant_tsc", REG_EXTENDED); + assert(rc == 0); + + FILE *cpuinfo = fopen("/proc/cpuinfo", "r"); + if (cpuinfo != NULL) { + while (fgets(buf, bufflen, cpuinfo) != NULL) { + if (regexec(&cpuGhzRegex, buf, nmatch, pmatch, 0) == 0) { + buf[pmatch[1].rm_eo] = '\0'; + double ghz = atof(&buf[pmatch[1].rm_so]); + mono_ticksPerMicrosecond = (long)(ghz * 1000); + break; + } + } + while (fgets(buf, bufflen, cpuinfo) != NULL) { + if (regexec(&constTscRegex, buf, nmatch, pmatch, 0) == 0) { + constantTsc = 1; + break; + } + } + + fclose(cpuinfo); + } + regfree(&cpuGhzRegex); + regfree(&constTscRegex); + + if (mono_ticksPerMicrosecond == 0) { + //fprintf(stderr, "monotonic: x86 linux, unable to determine clock rate"); + return; + } + if (!constantTsc) { + //fprintf(stderr, "monotonic: x86 linux, 'constant_tsc' flag not present"); + return; + } + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "X86 TSC @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_x86; +} +#endif + + +#if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__) +static long mono_ticksPerMicrosecond = 0; + +/* Read the clock value. */ +static inline uint64_t __cntvct() { + uint64_t virtual_timer_value; + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); + return virtual_timer_value; +} + +/* Read the Count-timer Frequency. */ +static inline uint32_t cntfrq_hz() { + uint64_t virtual_freq_value; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(virtual_freq_value)); + return (uint32_t)virtual_freq_value; /* top 32 bits are reserved */ +} + +static monotime getMonotonicUs_aarch64() { + return __cntvct() / mono_ticksPerMicrosecond; +} + +static void monotonicInit_aarch64() { + mono_ticksPerMicrosecond = (long)cntfrq_hz() / 1000L / 1000L; + if (mono_ticksPerMicrosecond == 0) { + fprintf(stderr, "monotonic: aarch64, unable to determine clock rate"); + return; + } + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "ARM CNTVCT @ %ld ticks/us", mono_ticksPerMicrosecond); + getMonotonicUs = getMonotonicUs_aarch64; +} +#endif + +static monotime getMonotonicUs_posix(void) { + /* clock_gettime() is specified in POSIX.1b (1993). Even so, some systems + * did not support this until much later. CLOCK_MONOTONIC is technically + * optional and may not be supported - but it appears to be universal. + * If this is not supported, provide a system-specific alternate version. */ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ((uint64_t)ts.tv_sec) * 1000000 + ts.tv_nsec / 1000; +} + +static void monotonicInit_posix() { + /* Ensure that CLOCK_MONOTONIC is supported. This should be supported + * on any reasonably current OS. If the assertion below fails, provide + * an appropriate alternate implementation. */ + struct timespec ts; + int rc = clock_gettime(CLOCK_MONOTONIC, &ts); + assert(rc == 0); + + snprintf(monotonic_info_string, sizeof(monotonic_info_string), + "POSIX clock_gettime"); + getMonotonicUs = getMonotonicUs_posix; +} + + + +const char * monotonicInit() { + #if defined(USE_PROCESSOR_CLOCK) && defined(__x86_64__) && defined(__linux__) + if (getMonotonicUs == NULL) monotonicInit_x86linux(); + #endif + + #if defined(USE_PROCESSOR_CLOCK) && defined(__aarch64__) + if (getMonotonicUs == NULL) monotonicInit_aarch64(); + #endif + + if (getMonotonicUs == NULL) monotonicInit_posix(); + + return monotonic_info_string; +} diff --git a/deps/wepoll/CMakeLists.txt b/deps/wepoll/CMakeLists.txt index a8b34112215fae0df7bcac034622015fd72d337b..a81fd782bbc4b05a1158273a7fcc6701bc4d980d 100644 --- a/deps/wepoll/CMakeLists.txt +++ b/deps/wepoll/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/zlib-1.2.11/CMakeLists.txt b/deps/zlib-1.2.11/CMakeLists.txt index a8750471d694bcfa98b711348861449e3fc7b23a..f83aa70085491fb6575c0a6bf93252192cddd040 100644 --- a/deps/zlib-1.2.11/CMakeLists.txt +++ b/deps/zlib-1.2.11/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_WINDOWS) diff --git a/deps/zlib-1.2.11/src/gzlib.c b/deps/zlib-1.2.11/src/gzlib.c index 381cdc2c7debea386cc4d70114d6b8823a521d16..4c135aabffffa7aa13d96f19b6bd2f059d7cec6e 100644 --- a/deps/zlib-1.2.11/src/gzlib.c +++ b/deps/zlib-1.2.11/src/gzlib.c @@ -5,6 +5,10 @@ #include #include "gzguts.h" +#ifndef O_BINARY + #define O_BINARY 0 +#endif + #if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) # define LSEEK _lseeki64 #else @@ -240,9 +244,9 @@ local gzFile gz_open(path, fd, mode) /* open the file with the appropriate flags (or just use fd) */ state->fd = fd > -1 ? fd : ( #ifdef WIDECHAR - fd == -2 ? _wopen(path, oflag, 0666) : + fd == -2 ? _wopen(path, oflag | O_BINARY, 0666) : #endif - open((const char *)path, oflag, 0666)); + open((const char *)path, oflag | O_BINARY, 0666)); if (state->fd == -1) { free(state->path); free(state); diff --git a/documentation/tdenginedocs-en/faq/index.html b/documentation/tdenginedocs-en/faq/index.html index a2fb8ba68ba909c2ad8273e6c5b8aa9e4a1d109b..2d716cd6dc1b4bc233dbe3a680fa17286f651c91 100644 --- a/documentation/tdenginedocs-en/faq/index.html +++ b/documentation/tdenginedocs-en/faq/index.html @@ -1,5 +1,5 @@ Documentation | Taos Data
Back

FAQ

-

1. When encoutered with the error "failed to connect to server", what can I do?

+

1. When encountered with the error "failed to connect to server", what can I do?

The client may encounter connection errors. Please follow the steps below for troubleshooting:

  1. On the server side, execute systemctl status taosd to check the status of taosd service. If taosd is not running, start it and retry connecting.
  2. diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..aba10a14e327ff104eb997b1ad6af29e3de6cad1 --- /dev/null +++ b/documentation20/cn/00.index/docs.md @@ -0,0 +1,153 @@ +# TDengine文档 + +TDengine是一个高效的存储、查询、分析时序大数据的平台,专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它,但建议您在使用前仔细阅读一遍下面的文档,特别是 [数据模型](/architecture) 与 [数据建模](/model)。除本文档之外,欢迎 [下载产品白皮书](https://www.taosdata.com/downloads/TDengine%20White%20Paper.pdf)。如需查阅TDengine 1.6 文档,请点击 [这里](https://www.taosdata.com/cn/documentation16/) 访问。 + +## [TDengine介绍](/evaluation) + +* [TDengine 简介及特色](/evaluation#intro) +* [TDengine 适用场景](/evaluation#scenes) +* [TDengine 性能指标介绍和验证方法](/evaluation#) + +## [立即开始](/getting-started) + +* [快捷安装](/getting-started#install):可通过源码、安装包或docker安装,三秒钟搞定 +* [轻松启动](/getting-started#start):使用systemctl 启停TDengine +* [命令行程序TAOS](/getting-started#console):访问TDengine的简便方式 +* [极速体验](/getting-started#demo):运行示例程序,快速体验高效的数据插入、查询 +* [支持平台列表](/getting-started#platforms):TDengine服务器和客户端支持的平台列表 + +## [整体架构](/architecture) + +* [数据模型](/architecture#model):关系型数据库模型,但要求每个采集点单独建表 +* [集群与基本逻辑单元](/architecture#cluster):吸取NoSQL优点,支持水平扩展,支持高可靠 +* [存储模型与数据分区、分片](/architecture#sharding):标签数据与时序数据完全分离,按vnode和时间两个维度对数据切分 +* [数据写入与复制流程](/architecture#replication):先写入WAL、之后写入缓存,再给应用确认,支持多副本 +* [缓存与持久化](/architecture#persistence):最新数据缓存在内存中,但落盘时采用列式存储、超高压缩比 +* [数据查询](/architecture#query):支持各种函数、时间轴聚合、插值、多表聚合 + +## [数据建模](/model) + +* [创建库](/model#create-db):为具有相似数据特征的数据采集点创建一个库 +* [创建超级表](/model#create-stable):为同一类型的数据采集点创建一个超级表 +* [创建表](/model#create-table):使用超级表做模板,为每一个具体的数据采集点单独建表 + +## [TAOS SQL](/taos-sql) + +* [支持的数据类型](/taos-sql#data-type):支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 +* [数据库管理](/taos-sql#management):添加、删除、查看数据库 +* [表管理](/taos-sql#table):添加、删除、查看、修改表 +* [超级表管理](/taos-sql#super-table):添加、删除、查看、修改超级表 +* [标签管理](/taos-sql#tags):增加、删除、修改标签 +* [数据写入](/taos-sql#insert):支持单表单条、多条、多表多条写入,支持历史数据写入 +* [数据查询](/taos-sql#select):支持时间段、值过滤、排序、查询结果手动分页等 +* [SQL函数](/taos-sql#functions):支持各种聚合函数、选择函数、计算函数,如avg, min, diff等 +* [时间维度聚合](/taos-sql#aggregation):将表中数据按照时间段进行切割后聚合,降维处理 +* [边界限制](/taos-sql#limitation):库、表、SQL等边界限制条件 +* [错误码](/taos-sql/error-code):TDengine 2.0 错误码以及对应的十进制码 + +## [高效写入数据](/insert) + +* [SQL写入](/insert#sql):使用SQL insert命令向一张或多张表写入单条或多条记录 +* [Prometheus写入](/insert#prometheus):配置Prometheus, 不用任何代码,将数据直接写入 +* [Telegraf写入](/insert#telegraf):配置Telegraf, 不用任何代码,将采集数据直接写入 +* [EMQ X Broker](/insert#emq):配置EMQ X,不用任何代码,就可将MQTT数据直接写入 +* [HiveMQ Broker](/insert#hivemq):配置HiveMQ,不用任何代码,就可将MQTT数据直接写入 + +## [高效查询数据](/queries) + +* [主要查询功能](/queries#queries):支持各种标准函数,设置过滤条件,时间段查询 +* [多表聚合查询](/queries#aggregation):使用超级表,设置标签过滤条件,进行高效聚合查询 +* [降采样查询值](/queries#sampling):按时间段分段聚合,支持插值 + +## [高级功能](/advanced-features) + +* [连续查询(Continuous Query)](/advanced-features#continuous-query):基于滑动窗口,定时自动的对数据流进行查询计算 +* [数据订阅(Publisher/Subscriber)](/advanced-features#subscribe):象典型的消息队列,应用可订阅接收到的最新数据 +* [缓存(Cache)](/advanced-features#cache):每个设备最新的数据都会缓存在内存中,可快速获取 +* [报警监测](/advanced-features#alert):根据配置规则,自动监测超限行为数据,并主动推送 + +## [连接器](/connector) + +* [C/C++ Connector](/connector#c-cpp):通过libtaos客户端的库,连接TDengine服务器的主要方法 +* [Java Connector(JDBC)](/connector/java):通过标准的JDBC API,给Java应用提供到TDengine的连接 +* [Python Connector](/connector#python):给Python应用提供一个连接TDengine服务器的驱动 +* [RESTful Connector](/connector#restful):提供一最简单的连接TDengine服务器的方式 +* [Go Connector](/connector#go):给Go应用提供一个连接TDengine服务器的驱动 +* [Node.js Connector](/connector#nodejs):给node应用提供一个连接TDengine服务器的驱动 +* [C# Connector](/connector#csharp):给C#应用提供一个连接TDengine服务器的驱动 +* [Windows客户端](https://www.taosdata.com/blog/2019/07/26/514.html):自行编译windows客户端,Windows环境的各种连接器都需要它 + +## [与其他工具的连接](/connections) + +* [Grafana](/connections#grafana):获取并可视化保存在TDengine的数据 +* [Matlab](/connections#matlab):通过配置Matlab的JDBC数据源访问保存在TDengine的数据 +* [R](/connections#r):通过配置R的JDBC数据源访问保存在TDengine的数据 +* [IDEA Database](https://www.taosdata.com/blog/2020/08/27/1767.html):通过IDEA 数据库管理工具可视化使用 TDengine + +## [TDengine集群的安装、管理](/cluster) + +* [准备工作](/cluster#prepare):部署环境前的几点注意事项 +* [创建第一个节点](/cluster#node-one):与快捷安装完全一样,非常简单 +* [创建后续节点](/cluster#node-other):配置新节点的taos.cfg, 在现有集群添加新的节点 +* [节点管理](/cluster#management):增加、删除、查看集群的节点 +* [Vnode 的高可用性](/cluster#high-availability):通过多副本的机制来提供 Vnode 的高可用性 +* [Mnode 的管理](/cluster#mnode):系统自动创建、无需任何人工干预 +* [负载均衡](/cluster#load-balancing):一旦节点个数或负载有变化,自动进行 +* [节点离线处理](/cluster#offline):节点离线超过一定时长,将从集群中剔除 +* [Arbitrator](/cluster#arbitrator):对于偶数个副本的情形,使用它可以防止split brain + +## [TDengine的运营和维护](/administrator) + +* [容量规划](/administrator#planning):根据场景,估算硬件资源 +* [容错和灾备](/administrator#tolerance):设置正确的WAL和数据副本数 +* [系统配置](/administrator#config):端口,缓存大小,文件块大小和其他系统配置 +* [用户管理](/administrator#user):添加、删除TDengine用户,修改用户密码 +* [数据导入](/administrator#import):可按脚本文件导入,也可按数据文件导入 +* [数据导出](/administrator#export):从shell按表导出,也可用taosdump工具做各种导出 +* [系统监控](/administrator#status):检查系统现有的连接、查询、流式计算,日志和事件等 +* [文件目录结构](/administrator#directories):TDengine数据文件、配置文件等所在目录 +* [参数限制与保留关键字](/administrator#keywords):TDengine的参数限制与保留关键字列表 + +## TDengine的技术设计 + +* [系统模块](/architecture/taosd):taosd的功能和模块划分 +* [数据复制](/architecture/replica):支持实时同步、异步复制,保证系统的High Availibility +* [技术博客](https://www.taosdata.com/cn/blog/?categories=3):更多的技术分析和架构设计文章 + +## 常用工具 + +* [TDengine样例导入工具](https://www.taosdata.com/blog/2020/01/18/1166.html) +* [TDengine性能对比测试工具](https://www.taosdata.com/blog/2020/01/18/1166.html) +* [IDEA数据库管理工具可视化使用TDengine](https://www.taosdata.com/blog/2020/08/27/1767.html) +* [基于eletron开发的跨平台TDengine图形化管理工具](https://github.com/skye0207/TDengineGUI) +* [DataX,支持TDengine的离线数据采集/同步工具](https://github.com/wgzhao/DataX)(文档:[读取插件](https://github.com/wgzhao/DataX/blob/master/docs/src/main/sphinx/reader/tdenginereader.md)、[写入插件](https://github.com/wgzhao/DataX/blob/master/docs/src/main/sphinx/writer/tdenginewriter.md)) + +## TDengine与其他数据库的对比测试 + +* [用InfluxDB开源的性能测试工具对比InfluxDB和TDengine](https://www.taosdata.com/blog/2020/01/13/1105.html) +* [TDengine与OpenTSDB对比测试](https://www.taosdata.com/blog/2019/08/21/621.html) +* [TDengine与Cassandra对比测试](https://www.taosdata.com/blog/2019/08/14/573.html) +* [TDengine与InfluxDB对比测试](https://www.taosdata.com/blog/2019/07/19/419.html) +* [TDengine与InfluxDB、OpenTSDB、Cassandra、MySQL、ClickHouse等数据库的对比测试报告](https://www.taosdata.com/downloads/TDengine_Testing_Report_cn.pdf) + +## 物联网大数据 + +* [物联网、工业互联网大数据的特点](https://www.taosdata.com/blog/2019/07/09/105.html) +* [物联网大数据平台应具备的功能和特点](https://www.taosdata.com/blog/2019/07/29/542.html) +* [通用大数据架构为什么不适合处理物联网数据?](https://www.taosdata.com/blog/2019/07/09/107.html) +* [物联网、车联网、工业互联网大数据平台,为什么推荐使用TDengine?](https://www.taosdata.com/blog/2019/07/09/109.html) + +## 培训和FAQ + +* [FAQ:常见问题与答案](/faq) +* [技术公开课:开源、高效的物联网大数据平台,TDengine内核技术剖析](https://www.taosdata.com/blog/2020/12/25/2126.html) +* [TDengine视频教程-快速上手](https://www.taosdata.com/blog/2020/11/11/1941.html) +* [TDengine视频教程-数据建模](https://www.taosdata.com/blog/2020/11/11/1945.html) +* [TDengine视频教程-集群搭建](https://www.taosdata.com/blog/2020/11/11/1961.html) +* [TDengine视频教程-Go Connector](https://www.taosdata.com/blog/2020/11/11/1951.html) +* [TDengine视频教程-JDBC Connector](https://www.taosdata.com/blog/2020/11/11/1955.html) +* [TDengine视频教程-NodeJS Connector](https://www.taosdata.com/blog/2020/11/11/1957.html) +* [TDengine视频教程-Python Connector](https://www.taosdata.com/blog/2020/11/11/1963.html) +* [TDengine视频教程-RESTful Connector](https://www.taosdata.com/blog/2020/11/11/1965.html) +* [TDengine视频教程-“零”代码运维监控](https://www.taosdata.com/blog/2020/11/11/1959.html) +* [应用案例:一些使用实例来解释如何使用TDengine](https://www.taosdata.com/cn/blog/?categories=4) diff --git a/documentation20/webdocs/markdowndocs/Evaluation-ch.md b/documentation20/cn/01.evaluation/docs.md similarity index 97% rename from documentation20/webdocs/markdowndocs/Evaluation-ch.md rename to documentation20/cn/01.evaluation/docs.md index fa6cec6e488d144a7009dc52772987380d5065da..0ae2106ff2a63696dc8bbc51d25bbf5e811ef561 100644 --- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md +++ b/documentation20/cn/01.evaluation/docs.md @@ -1,6 +1,6 @@ # TDengine 介绍 -## TDengine 简介 +## TDengine 简介 TDengine是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品,它不依赖任何第三方软件,也不是优化或包装了一个开源的数据库或流式计算产品,而是在吸取众多传统关系型数据库、NoSQL数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品,在时序空间大数据处理上,有着自己独到的优势。 @@ -15,10 +15,11 @@ TDengine的模块之一是时序数据库。但除此之外,为减少研发的 采用TDengine,可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是,因充分利用了物联网时序数据的特点,它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。 -
    +![TDengine技术生态图](page://images/eco_system.png)
    图 1. TDengine技术生态图
    -## TDengine 总体适用场景 + +## TDengine 总体适用场景 作为一个IOT大数据平台,TDengine的典型适用场景是在IOT范畴,而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统,比如CRM,ERP等,不在本文讨论范围内。 diff --git a/documentation20/webdocs/markdowndocs/Getting Started-ch.md b/documentation20/cn/02.getting-started/docs.md similarity index 72% rename from documentation20/webdocs/markdowndocs/Getting Started-ch.md rename to documentation20/cn/02.getting-started/docs.md index b53c014ba64fb203cb8e6943297fe6117034385c..43392e032518ae3245edece20f4f4168e1108db0 100644 --- a/documentation20/webdocs/markdowndocs/Getting Started-ch.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -1,10 +1,10 @@ # 立即开始 -## 快捷安装 +## 快捷安装 TDengine软件分为服务器、客户端和报警模块三部分,目前2.0版服务器仅能在Linux系统上安装和运行,后续会支持Windows、mac OS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64,后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。 -### 通过源码安装 +### 通过源码安装 请参考我们的[TDengine github主页](https://github.com/taosdata/TDengine)下载源码并安装. @@ -12,17 +12,15 @@ TDengine软件分为服务器、客户端和报警模块三部分,目前2.0版 请参考[TDengine官方Docker镜像的发布、下载和使用](https://www.taosdata.com/blog/2020/05/13/1509.html) -### 通过安装包安装 +### 通过安装包安装 TDengine的安装非常简单,从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器,我们提供三种安装包,您可以根据需要选择: -- TDengine-server-2.0.10.0-Linux-x64.rpm (4.2M) -- TDengine-server-2.0.10.0-Linux-x64.deb (2.7M) -- TDengine-server-2.0.10.0-Linux-x64.tar.gz (4.5M) +安装包下载在[这里](https://www.taosdata.com/cn/getting-started/#通过安装包安装)。 -具体的安装过程,请参见TDengine多种安装包的安装和卸载以及视频教程。 +具体的安装过程,请参见[TDengine多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html)以及[视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。 -## 轻松启动 +## 轻松启动 安装成功后,用户可使用`systemctl`命令来启动TDengine的服务进程。 @@ -52,8 +50,7 @@ $ systemctl status taosd 如果系统中不支持systemd,也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。 - -## TDengine命令行程序 +## TDengine命令行程序 执行TDengine命令行程序,您只要在Linux终端执行`taos`即可。 @@ -61,7 +58,7 @@ $ systemctl status taosd $ taos ``` -如果TDengine终端连接服务成功,将会打印出欢迎消息和版本信息。如果失败,则会打印错误消息出来(请参考[FAQ](https://www.taosdata.com/cn/faq/)来解决终端连接服务端失败的问题)。TDengine终端的提示符号如下: +如果TDengine终端连接服务成功,将会打印出欢迎消息和版本信息。如果失败,则会打印错误消息出来(请参考[FAQ](https://www.taosdata.com/cn/documentation/faq/)来解决终端连接服务端失败的问题)。TDengine终端的提示符号如下: ```cmd taos> @@ -117,7 +114,8 @@ taos> source ; - ctrl+c 中止正在进行中的查询 - 执行`RESET QUERY CACHE`清空本地缓存的表的schema -## TDengine 极速体验 + +## TDengine 极速体验 启动TDengine的服务,在Linux终端执行taosdemo @@ -164,7 +162,6 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); **Note:** taosdemo命令本身带有很多选项,配置表的数目、记录条数等等,请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。 - ## 客户端和报警模块 如果客户端和服务端运行在不同的电脑上,可以单独安装客户端。Linux和Windows安装包如下: @@ -178,24 +175,24 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); - TDengine-alert-2.0.10.0-Linux-x64.tar.gz (8.1M) - -## **支持平台列表** +## 支持平台列表 ### TDengine服务器支持的平台列表 -| | **CentOS** **6/7/8** | **Ubuntu** **16/18/20** | **Other Linux** | **统信****UOS** | **银河****/****中标麒麟** | **凝思** **V60/V80** | -| -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | -| X64 | ● | ● | | ○ | ● | ● | -| 树莓派ARM32 | | ● | ● | | | | -| 龙芯MIPS64 | | | ● | | | | -| 鲲鹏 ARM64 | | ○ | ○ | | ● | | -| 申威 Alpha64 | | | ○ | ● | | | -| 飞腾ARM64 | | ○优麒麟 | | | | | -| 海光X64 | ● | ● | ● | ○ | ● | ● | -| 瑞芯微ARM64/32 | | | ○ | | | | -| 全志ARM64/32 | | | ○ | | | | -| 炬力ARM64/32 | | | ○ | | | | -| TI ARM32 | | | ○ | | | | +| | **CentOS 6/7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | +| -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | --------------------- | +| X64 | ● | ● | | ○ | ● | ● | ● | +| 树莓派 ARM32 | | ● | ● | | | | | +| 龙芯 MIPS64 | | | ● | | | | | +| 鲲鹏 ARM64 | | ○ | ○ | | ● | | | +| 申威 Alpha64 | | | ○ | ● | | | | +| 飞腾 ARM64 | | ○ 优麒麟 | | | | | | +| 海光 X64 | ● | ● | ● | ○ | ● | ● | | +| 瑞芯微 ARM64/32 | | | ○ | | | | | +| 全志 ARM64/32 | | | ○ | | | | | +| 炬力 ARM64/32 | | | ○ | | | | | +| TI ARM32 | | | ○ | | | | | +| 华为云 ARM64 | | | | | | | ● | 注: ● 表示经过官方测试验证, ○ 表示非官方测试验证。 @@ -207,7 +204,7 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); 对照矩阵如下: -| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS ** **龙芯** | **Alpha ** **申威** | **X64 ** **海光** | +| **CPU** | **X64 64bit** | | | **X86 32bit** | **ARM64** | **ARM32** | **MIPS 龙芯** | **Alpha 申威** | **X64 海光** | | ----------- | --------------- | --------- | --------- | --------------- | --------- | --------- | ------------------- | -------------------- | ------------------ | | **OS** | **Linux** | **Win64** | **Win32** | **Win32** | **Linux** | **Linux** | **Linux** | **Linux** | **Linux** | | **C/C++** | ● | ● | ● | ○ | ● | ● | ● | ● | ● | @@ -220,5 +217,5 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); 注: ● 表示经过官方测试验证, ○ 表示非官方测试验证。 -请跳转到 [连接器 ](https://www.taosdata.com/cn/documentation/connector)查看更详细的信息。 +请跳转到 [连接器](https://www.taosdata.com/cn/documentation/connector)查看更详细的信息。 diff --git a/documentation20/webdocs/markdowndocs/taosd-ch.md b/documentation20/cn/03.architecture/01.taosd/docs.md similarity index 97% rename from documentation20/webdocs/markdowndocs/taosd-ch.md rename to documentation20/cn/03.architecture/01.taosd/docs.md index 08be0c163e7076b58f03ff8ea3165e902a80fe64..66d51ed2dc2ea1546ab167cad680c20b3fa9729c 100644 --- a/documentation20/webdocs/markdowndocs/taosd-ch.md +++ b/documentation20/cn/03.architecture/01.taosd/docs.md @@ -3,9 +3,10 @@ 逻辑上,TDengine系统包含dnode, taosc和App,dnode是服务器侧执行代码taosd的一个运行实例,因此taosd是TDengine的核心,本文对taosd的设计做一简单的介绍,模块内的实现细节请见其他文档。 ## 系统模块图 + taosd包含rpc, dnode, vnode, tsdb, query, cq, sync, wal, mnode, http, monitor等模块,具体如下图: -
    +![modules.png](page://images/architecture/modules.png) taosd的启动入口是dnode模块,dnode然后启动其他模块,包括可选配置的http, monitor模块。taosc或dnode之间交互的消息都是通过rpc模块进行,dnode模块根据接收到的消息类型,将消息分发到vnode或mnode的消息队列,或由dnode模块自己消费。dnode的工作线程(worker)消费消息队列里的消息,交给mnode或vnode进行处理。下面对各个模块做简要说明。 @@ -40,13 +41,14 @@ RPC模块还提供数据压缩功能,如果数据包的字节数超过系统 taosd的消息消费由dnode通过读写线程池进行控制,是系统的中枢。该模块内的结构体图如下: -
    +![dnode.png](page://images/architecture/dnode.png) ## VNODE模块 vnode是一独立的数据存储查询逻辑单元,但因为一个vnode只能容许一个DB,因此vnode内部没有account, DB, user等概念。为实现更好的模块化、封装以及未来的扩展,它有很多子模块,包括负责存储的TSDB,负责查询的Query, 负责数据复制的sync,负责数据库日志的的wal, 负责连续查询的cq(continuous query), 负责事件触发的流计算的event等模块,这些子模块只与vnode模块发生关系,与其他模块没有任何调用关系。模块图如下: -
    +![vnode.png](page://images/architecture/vnode.png) + vnode模块向下,与dnodeVRead,dnodeVWrite发生互动,向上,与子模块发生互动。它主要的功能有: - 协调各个子模块的互动。各个子模块之间都不直接调用,都需要通过vnode模块进行; @@ -68,30 +70,37 @@ mnode是整个系统的大脑,负责整个系统的资源调度,负责meta d mnode里还负责account, user, DB, stable, table, vgroup, dnode的创建、删除与更新。mnode不仅把这些entity的meta data保存在内存,还做持久化存储。但为节省内存,各个表的标签值不保存在mnode(保存在vnode),而且子表不维护自己的schema, 而是与stable共享。为减小mnode的查询压力,taosc会缓存table、stable的schema。对于查询类的操作,各个slave mnode也可以提供,以减轻master压力。 ## TSDB模块 + TSDB模块是VNODE中的负责快速高并发地存储和读取属于该VNODE的表的元数据及采集的时序数据的引擎。除此之外,TSDB还提供了表结构的修改、表标签值的修改等功能。TSDB提供API供VNODE和Query等模块调用。TSDB中存储了两类数据,1:元数据信息;2:时序数据 ### 元数据信息 + TSDB中存储的元数据包含属于其所在的VNODE中表的类型,schema的定义等。对于超级表和超级表下的子表而言,又包含了tag的schema定义以及子表的tag值等。对于元数据信息而言,TSDB就相当于一个全内存的KV型数据库,属于该VNODE的表对象全部在内存中,方便快速查询表的信息。除此之外,TSDB还对其中的子表,按照tag的第一列取值做了全内存的索引,大大加快了对于标签的过滤查询。TSDB中的元数据的最新状态在落盘时,会以追加(append-only)的形式,写入到meta文件中。meta文件只进行追加操作,即便是元数据的删除,也会以一条记录的形式写入到文件末尾。TSDB也提供了对于元数据的修改操作,如表schema的修改,tag schema的修改以及tag值的修改等。 ### 时序数据 + 每个TSDB在创建时,都会事先分配一定量的内存缓冲区,且内存缓冲区的大小可配可修改。表采集的时序数据,在写入TSDB时,首先以追加的方式写入到分配的内存缓冲区中,同时建立基于时间戳的内存索引,方便快速查询。当内存缓冲区的数据积累到一定的程度时(达到内存缓冲区总大小的1/3),则会触发落盘操作,将缓冲区中的数据持久化到硬盘文件上。时序数据在内存缓冲区中是以行(row)的形式存储的。 而时序数据在写入到TSDB的数据文件时,是以列(column)的形式存储的。TSDB中的数据文件包含多个数据文件组,每个数据文件组中又包含.head、.data和.last三个文件,如(v2f1801.head、v2f1801.data、v2f1801.last)数据文件组。TSDB中的数据文件组是按照时间跨度进行分片的,默认是10天一个文件组,且可通过配置文件及建库选项进行配置。分片的数据文件组又按照编号递增排列,方便快速定位某一时间段的时序数据,高效定位数据文件组。时序数据在TSDB的数据文件中是以块的形式进行列式存储的,每个块中只包含一张表的数据,且数据在一个块中是按照时间顺序递增排列的。在一个数据文件组中,.head文件负责存储数据块的索引及统计信息,如每个块的位置,压缩算法,时间戳范围等。存储在.head文件中一张表的索引信息是按照数据块中存储的数据的时间递增排列的,方便进行折半查找等工作。.head和.last文件是存储真实数据块的文件,若数据块中的数据累计到一定程度,则会写入.data文件中,否则,会写入.last文件中,等待下次落盘时合并数据写入.data文件中,从而大大减少文件中块的个数,避免数据的过度碎片化。 ## Query模块 + 该模块负责整体系统的查询处理。客户端调用该该模块进行SQL语法解析,并将查询或写入请求发送到vnode,同时负责针对超级表的查询进行二阶段的聚合操作。在Vnode端,该模块调用TSDB模块读取系统中存储的数据进行查询处理。Query模块还定义了系统能够支持的全部查询函数,查询函数的实现机制与查询框架无耦合,可以在不修改查询流程的情况下动态增加查询函数。详细的设计请参见《TDengine 2.0查询模块设计》。 ## SYNC模块 + 该模块实现数据的多副本复制,包括vnode与mnode的数据复制,支持异步和同步两种复制方式,以满足meta data与时序数据不同复制的需求。因为它为mnode与vnode共享,系统为mnode副本预留了一个特殊的vgroup ID:1。因此vnode group的ID是从2开始的。 -每个vnode/mnode模块实例会有一对应的sync模块实例,他们是一一对应的。详细设计请见TDengine 2.0 数据复制模块设计 +每个vnode/mnode模块实例会有一对应的sync模块实例,他们是一一对应的。详细设计请见[TDengine 2.0 数据复制模块设计](https://www.taosdata.com/cn/documentation/architecture/replica/) ## WAL模块 + 该模块负责将新插入的数据写入write ahead log(WAL), 为vnode, mnode共享。以保证服务器crash或其他故障,能从WAL中恢复数据。 每个vnode/mnode模块实例会有一对应的wal模块实例,是完全一一对应的。WAL的落盘操作由两个参数walLevel, fsync控制。看具体场景,如果要100%保证数据不会丢失,需要将walLevel配置为2,fsync设置为0,每条数据插入请求,都会实时落盘后,才会给应用确认 ## HTTP模块 + 该模块负责处理系统对外的RESTful接口,可以通过配置,由dnode启动或停止。 该模块将接收到的RESTful请求,做了各种合法性检查后,将其变成标准的SQL语句,通过taosc的异步接口,将请求发往整个系统中的任一dnode。收到处理后的结果后,再翻译成HTTP协议,返回给应用。 @@ -99,6 +108,7 @@ TSDB中存储的元数据包含属于其所在的VNODE中表的类型,schema 如果HTTP模块启动,就意味着启动了一个taosc的实例。任一一个dnode都可以启动该模块,以实现对RESTful请求的分布式处理。 ## Monitor模块 + 该模块负责检测一个dnode的运行状态,可以通过配置,由dnode启动或停止。原则上,每个dnode都应该启动一个monitor实例。 Monitor采集TDengine里的关键操作,比如创建、删除、更新账号、表、库等,而且周期性的收集CPU、内存、网络等资源的使用情况(采集周期由系统配置参数monitorInterval控制)。获得这些数据后,monitor模块将采集的数据写入系统的日志库(DB名字由系统配置参数monitorDbName控制)。 diff --git a/documentation20/webdocs/markdowndocs/replica-ch.md b/documentation20/cn/03.architecture/02.replica/docs.md similarity index 95% rename from documentation20/webdocs/markdowndocs/replica-ch.md rename to documentation20/cn/03.architecture/02.replica/docs.md index 4d714fb5502534521e47d761e24af67b31248f51..8e1b1e3ab1513fbeaa5b9b805263485a13483b9b 100644 --- a/documentation20/webdocs/markdowndocs/replica-ch.md +++ b/documentation20/cn/03.architecture/02.replica/docs.md @@ -10,13 +10,13 @@ TDengine面向的是物联网场景,需要支持数据的实时复制,来最 数据复制是与数据存储(写入、读取)密切相关的,但两者又是相对独立,可以完全脱耦的。在TDengine系统中,有两种不同类型的数据,一种是时序数据,由TSDB模块负责;一种是元数据(Meta Data), 由MNODE负责。这两种性质不同的数据都需要同步功能。数据复制模块通过不同的实例启动配置参数,为这两种类型数据都提供同步功能。 -在阅读本文之前,请先阅读《TDengine 2.0 整体架构》,了解TDengine的集群设计和基本概念 +在阅读本文之前,请先阅读《[TDengine 2.0 整体架构](https://www.taosdata.com/cn/documentation/architecture/)》,了解TDengine的集群设计和基本概念 特别注明:本文中提到数据更新操作包括数据的增加、删除与修改。 ## 基本概念和定义 -TDengine里存在vnode, mnode, vnode用来存储时序数据,mnode用来存储元数据。但从同步数据复制的模块来看,两者没有本质的区别,因此本文里的虚拟节点不仅包括vnode, 也包括mnode, vgoup也指mnode group, 除非特别注明。 +TDengine里存在vnode, mnode, vnode用来存储时序数据,mnode用来存储元数据。但从同步数据复制的模块来看,两者没有本质的区别,因此本文里的虚拟节点不仅包括vnode, 也包括mnode, vgroup也指mnode group, 除非特别注明。 **版本(version)**: @@ -90,7 +90,7 @@ TDengine采取的是Master-Slave模式进行同步,与流行的RAFT一致性 具体的流程图如下: -
    +![replica-master.png](page://images/architecture/replica-master.png) 选择Master的具体规则如下: @@ -105,7 +105,7 @@ TDengine采取的是Master-Slave模式进行同步,与流行的RAFT一致性 如果vnode A是master, vnode B是slave, vnode A能接受客户端的写请求,而vnode B不能。当vnode A收到写的请求后,遵循下面的流程: -
    +![replica-forward.png](page://images/architecture/replica-forward.png) 1. 应用对写请求做基本的合法性检查,通过,则给改请求包打上一个版本号(version, 单调递增) 2. 应用将打上版本号的写请求封装一个WAL Head, 写入WAL(Write Ahead Log) @@ -128,19 +128,19 @@ TDengine采取的是Master-Slave模式进行同步,与流行的RAFT一致性 2. 任何一个数据文件(file)有名字、大小,还有一个magic number。只有文件名、大小与magic number一致时,两个文件才判断是一样的,无需同步。Magic number可以是checksum, 也可以是简单的文件大小。怎么计算magic,换句话说,如何检测数据文件是否有效,完全由应用决定。 3. 文件名的处理有点复杂,因为每台服务器的路径可能不一致。比如node A的TDengine的数据文件存放在 /etc/taos目录下,而node B的数据存放在 /home/jhtao目录下。因此同步模块需要应用在启动一个同步实例时提供一个path,这样两台服务器的绝对路径可以不一样,但仍然可以做对比,做同步。 4. 当sync模块调用回调函数getFileInfo获得数据文件信息时,有如下的规则 - 1. index 为0,表示获取最老的文件,同时修改index返回给sync模块。如果index不为0,表示获取指定位置的文件。 - 2. 如果name为空,表示sync想获取位于index位置的文件信息,包括magic, size。Master节点会这么调用 - 3. 如果name不为空,表示sync想获取指定文件名和index的信息,slave节点会这么调用 - 4. 如果某个index的文件不存在,magic返回0,表示文件已经是最后一个。因此整个系统里,文件的index必须是连续的一段整数。 + * index 为0,表示获取最老的文件,同时修改index返回给sync模块。如果index不为0,表示获取指定位置的文件。 + * 如果name为空,表示sync想获取位于index位置的文件信息,包括magic, size。Master节点会这么调用 + * 如果name不为空,表示sync想获取指定文件名和index的信息,slave节点会这么调用 + * 如果某个index的文件不存在,magic返回0,表示文件已经是最后一个。因此整个系统里,文件的index必须是连续的一段整数。 5. 当sync模块调用回调函数getWalInfo获得wal信息时,有如下规则 - 1. index为0,表示获得最老的WAL文件, 返回时,index更新为具体的数字 - 2. 如果返回0,表示这是最新的一个WAL文件,如果返回值是1,表示后面还有更新的WAL文件 - 3. 返回的文件名为空,那表示没有WAL文件 + * index为0,表示获得最老的WAL文件, 返回时,index更新为具体的数字 + * 如果返回0,表示这是最新的一个WAL文件,如果返回值是1,表示后面还有更新的WAL文件 + * 返回的文件名为空,那表示没有WAL文件 6. 无论是getFileInfo, 还是getWalInfo, 只要获取出错(不是文件不存在),返回-1即可,系统会报错,停止同步 整个数据恢复流程分为两大步骤,第一步,先恢复archived data(file), 然后恢复wal。具体流程如下: -
    +![replica-forward.png](page://images/architecture/replica-forward.png) 1. 通过已经建立的TCP连接,发送sync req给master节点 2. master收到sync req后,以client的身份,向vnode B主动建立一新的专用于同步的TCP连接(syncFd) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/cn/03.architecture/docs.md similarity index 90% rename from documentation20/webdocs/markdowndocs/architecture-ch.md rename to documentation20/cn/03.architecture/docs.md index 47fb8094b7379b9613f204b3d8c773dbff3bb4d3..87553fa8ad9760ecdb6d1667823d336189542331 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/cn/03.architecture/docs.md @@ -1,7 +1,9 @@ # 数据模型和整体架构 -## 数据模型 +## 数据模型 + ### 物联网典型场景 + 在典型的物联网、车联网、运维监测场景中,往往有多种不同类型的数据采集设备,采集一个到多个不同的物理量。而同一种采集设备类型,往往又有多个具体的采集设备分布在不同的地点。大数据处理系统就是要将各种采集的数据汇总,然后进行计算和分析。对于同一类设备,其采集的数据都是很规则的。以智能电表为例,假设每个智能电表采集电流、电压、相位三个量,其采集的数据类似如下的表格:
    @@ -103,6 +105,7 @@ 每一条记录都有设备ID,时间戳,采集的物理量(如上图中的电流、电压、相位),还有与每个设备相关的静态标签(如上述表一中的位置Location和分组groupId)。每个设备是受外界的触发,或按照设定的周期采集数据。采集的数据点是时序的,是一个数据流。 ### 数据特征 + 除时序特征外,仔细研究发现,物联网、车联网、运维监测类数据还具有很多其他明显的特征: 1. 数据高度结构化; @@ -119,9 +122,11 @@ 充分利用上述特征,TDengine 采取了经特殊优化的存储和计算设计来处理时序数据,它将系统处理能力显著提高,同时大幅降低了系统运维的复杂度。 ### 关系型数据库模型 + 因为采集的数据一般是结构化数据,同时为降低学习门槛,TDengine采用传统的关系型数据库模型管理数据。因此用户需要先创建库,然后创建表,之后才能插入或查询数据。TDengine采用的是结构化存储,而不是NoSQL的key-value存储。 ### 一个数据采集点一张表 + 为充分利用其数据的时序性和其他数据特点,TDengine要求**对每个数据采集点单独建表**(比如有一千万个智能电表,就需创建一千万张表,上述表格中的d1001, d1002, d1003, d1004都需单独建表),用来存储这个采集点所采集的时序数据。这种设计有几大优点: 1. 能保证一个采集点的数据在存储介质上是以块为单位连续存储的。如果读取一个时间段的数据,它能大幅减少随机读取操作,成数量级的提升读取和查询速度。 @@ -133,56 +138,65 @@ TDengine 建议用数据采集点的名字(如上表中的D1001)来做表名。每个数据采集点可能同时采集多个物理量(如上表中的curent, voltage, phase),每个物理量对应一张表中的一列,数据类型可以是整型、浮点型、字符串等。除此之外,表的第一列必须是时间戳,即数据类型为 timestamp。对采集的数据,TDengine将自动按照时间戳建立索引,但对采集的物理量不建任何索引。数据用列式存储方式保存。 ### 超级表:同一类型数据采集点的集合 + 由于一个数据采集点一张表,导致表的数量巨增,难以管理,而且应用经常需要做采集点之间的聚合操作,聚合的操作也变得复杂起来。为解决这个问题,TDengine引入超级表(Super Table,简称为STable)的概念。 超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点,其表的结构是完全一样的,但每个表(数据采集点)的静态属性(标签)是不一样的。描述一个超级表(某一特定类型的数据采集点的结合),除需要定义采集量的表结构之外,还需要定义其标签的schema,标签的数据类型可以是整数、浮点数、字符串,标签可以有多个,可以事后增加、删除或修改。 如果整个系统有N个不同类型的数据采集点,就需要建立N个超级表。 在TDengine的设计里,**表用来代表一个具体的数据采集点,超级表用来代表一组相同类型的数据采集点集合**。当为某个具体数据采集点创建表时,用户使用超级表的定义做模板,同时指定该具体采集点(表)的标签值。与传统的关系型数据库相比,表(一个数据采集点)是带有静态标签的,而且这些标签可以事后增加、删除、修改。**一张超级表包含有多张表,这些表具有相同的时序数据schema,但带有不同的标签值**。 -当对多个具有相同数据类型的数据采集点进行聚合操作时,TDengine将先把满足标签过滤条件的表从超级表的中查找出来,然后再扫描这些表的时序数据,进行聚合操作,这样能将需要扫描的数据集大幅减少,从而大幅提高聚合计算的性能。 +当对多个具有相同数据类型的数据采集点进行聚合操作时,TDengine会先把满足标签过滤条件的表从超级表中找出来,然后再扫描这些表的时序数据,进行聚合操作,这样需要扫描的数据集会大幅减少,从而显著提高聚合计算的性能。 + +## 集群与基本逻辑单元 -## 集群与基本逻辑单元 TDengine 的设计是基于单个硬件、软件系统不可靠,基于任何单台计算机都无法提供足够计算能力和存储能力处理海量数据的假设进行设计的。因此 TDengine 从研发的第一天起,就按照分布式高可靠架构进行设计,是支持水平扩展的,这样任何单台或多台服务器发生硬件故障或软件错误都不影响系统的可用性和可靠性。同时,通过节点虚拟化并辅以自动化负载均衡技术,TDengine 能最高效率地利用异构集群中的计算和存储资源降低硬件投资。 ### 主要逻辑单元 + TDengine 分布式架构的逻辑结构图如下: -
    + +![TDengine架构示意图](page://images/architecture/structure.png)
    图 1 TDengine架构示意图
    + 一个完整的 TDengine 系统是运行在一到多个物理节点上的,逻辑上,它包含数据节点(dnode)、TDengine应用驱动(taosc)以及应用(app)。系统中存在一到多个数据节点,这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。 -**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机,可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯,如果不了解FQDN,请看博文《一篇文章说清楚TDengine的FQDN》。 +**物理节点(pnode):** pnode是一独立运行、拥有自己的计算、存储和网络能力的计算机,可以是安装有OS的物理机、虚拟机或Docker容器。物理节点由其配置的 FQDN(Fully Qualified Domain Name)来标识。TDengine完全依赖FQDN来进行网络通讯,如果不了解FQDN,请看博文[《一篇文章说清楚TDengine的FQDN》](https://www.taosdata.com/blog/2020/09/11/1824.html)。 **数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例,一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE),零或者至多一个逻辑的管理节点(mnode)。dnode在系统中的唯一标识由实例的End Point (EP )决定。EP是dnode所在物理节点的FQDN (Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口,一个物理节点(一台物理机、虚拟机或容器)可以运行多个实例,或有多个数据节点。 **虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡,防止数据过热或倾斜,数据节点被虚拟化成多个虚拟节点(vnode,图中V2, V3, V4等)。每个 vnode 都是一个相对独立的工作单元,是时序数据存储的基本单元,具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表(数据采集点)。当创建一张新表时,系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB,但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外,也保存有所包含的表的schema、标签值等。一个虚拟节点由所属的数据节点的EP,以及所属的VGroup ID在系统内唯一标识,由管理节点创建并管理。 -**管理节点(mnode):** 一个虚拟的逻辑单元,负责所有数据节点运行状态的监控和维护,以及节点之间的负载均衡(图中M)。同时,管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理,因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode,它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理,而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成,无需人工干预。每个dnode上至多有一个mnode,由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。 +**管理节点(mnode):** 一个虚拟的逻辑单元,负责所有数据节点运行状态的监控和维护,以及节点之间的负载均衡(图中M)。同时,管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理,因此也称为 Meta Node。TDengine 集群中可配置多个(开源版最多不超过3个) mnode,它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理,而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成,无需人工干预。每个dnode上至多有一个mnode,由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。 **虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行,系统采用异步复制的方式将数据同步到 slave vnode,这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N,系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定,缺省为1。使用 TDengine 的多副本特性,可以不再需要昂贵的磁盘阵列等存储设备,就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理,并且由管理节点分配一个系统唯一的ID,VGroup ID。如果两个虚拟节点的vnode group ID相同,说明他们属于同一个组,数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的,容许只有一个,也就是没有数据复制。VGroup ID是永远不变的,即使一个虚拟节点组被删除,它的ID也不会被收回重复利用。 **TAOSC:** taosc是TDengine给应用提供的驱动程序(driver),负责处理应用与集群的接口交互,提供C/C++语言原生接口,内嵌于JDBC、C#、Python、Go、Node.js语言连接库里。应用都是通过taosc而不是直接连接集群中的数据节点与整个集群进行交互的。这个模块负责获取并缓存元数据;将插入、查询等请求转发到正确的数据节点;在把结果返回给应用时,还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, C/C++/C#/Python/Go/Node.js接口而言,这个模块是在应用所处的物理节点上运行。同时,为支持全分布式的RESTful接口,taosc在TDengine集群的每个dnode上都有一运行实例。 ### 节点之间的通讯 + **通讯方式:**TDengine系统的各个数据节点之间,以及应用驱动与各数据节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景,数据写入的包一般不大,因此TDengine 除采用TCP做传输之外,还采用UDP方式,因为UDP 更加高效,而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制,以确保UDP的可靠传输。对于数据量不到15K的数据包,采取UDP的方式进行传输,超过15K的,或者是查询类的操作,自动采取TCP的方式进行传输。同时,TDengine根据配置和数据包,会自动对数据进行压缩/解压缩,数字签名/认证等处理。对于数据节点之间的数据复制,只采用TCP方式进行数据传输。 **FQDN配置**:一个数据节点有一个或多个FQDN,可以在系统配置文件taos.cfg通过参数“fqdn"进行指定,如果没有指定,系统将自动获取计算机的hostname作为其FQDN。如果节点没有配置FQDN,可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP,因为IP地址可变,一旦变化,将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN,需要保证DNS服务正常工作,或者在节点以及应用所在的节点配置好hosts文件。 -**端口配置:**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定,对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口,是serverPort+10. 为支持多线程高效的处理UDP数据,每个对内和对外的UDP连接,都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10,总共11个TCP/UDP端口。使用时,需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 +**端口配置:**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定,对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口,是serverPort+10. 为支持多线程高效的处理UDP数据,每个对内和对外的UDP连接,都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10,总共11个TCP/UDP端口。(另外还可能有 RESTful、Arbitrator 所使用的端口,那样的话就一共是 13 个。)使用时,需要确保防火墙将这些端口打开,以备使用。每个数据节点可以配置不同的serverPort。 **集群对外连接:** TDengine集群可以容纳单个、多个甚至几千个数据节点。应用只需要向集群中任何一个数据节点发起连接即可,连接需要提供的网络参数是一数据节点的End Point(FQDN加配置的端口号)。通过命令行CLI启动应用taos时,可以通过选项-h来指定数据节点的FQDN, -P来指定其配置的端口号,如果端口不配置,将采用TDengine的系统配置参数serverPort。 -**集群内部通讯**: 各个数据节点之间通过TCP/UDP进行连接。一个数据节点启动时,将获取mnode所在的dnode的EP信息,然后与系统中的mnode建立起连接,交换信息。获取mnode的EP信息有三步,1:检查mnodeEpList文件是否存在,如果不存在或不能正常打开获得mnode EP信息,进入第二步;2:检查系统配置文件taos.cfg, 获取mnode EP配置参数first, second,如果不存在或者taos.cfg里没有这两个配置参数,或无效,进入第三步;3:将自己的EP设为mnode EP, 并独立运行起来。获取mnode EP列表后,数据节点发起连接,如果连接成功,则成功加入进工作的集群,如果不成功,则尝试mnode EP列表中的下一个。如果都尝试了,但连接都仍然失败,则休眠几秒后,再进行尝试。 +**集群内部通讯**: 各个数据节点之间通过TCP/UDP进行连接。一个数据节点启动时,将获取mnode所在的dnode的EP信息,然后与系统中的mnode建立起连接,交换信息。获取mnode的EP信息有三步,1:检查mnodeEpSet文件是否存在,如果不存在或不能正常打开获得mnode EP信息,进入第二步;2:检查系统配置文件taos.cfg, 获取节点配置参数firstEp, secondEp,(这两个参数指定的节点可以是不带mnode的普通节点,这样的话,节点被连接时会尝试重定向到mnode节点)如果不存在或者taos.cfg里没有这两个配置参数,或无效,进入第三步;3:将自己的EP设为mnode EP, 并独立运行起来。获取mnode EP列表后,数据节点发起连接,如果连接成功,则成功加入进工作的集群,如果不成功,则尝试mnode EP列表中的下一个。如果都尝试了,但连接都仍然失败,则休眠几秒后,再进行尝试。 **MNODE的选择:** TDengine逻辑上有管理节点,但没有单独的执行代码,服务器侧只有一套执行代码taosd。那么哪个数据节点会是管理节点呢?这是系统自动决定的,无需任何人工干预。原则如下:一个数据节点启动时,会检查自己的End Point, 并与获取的mnode EP List进行比对,如果在其中,该数据节点认为自己应该启动mnode模块,成为mnode。如果自己的EP不在mnode EP List里,则不启动mnode模块。在系统的运行过程中,由于负载均衡、宕机等原因,mnode有可能迁移至新的dnode,但一切都是透明的,无需人工干预,配置参数的修改,是mnode自己根据资源做出的决定。 -**新数据节点的加入**:系统有了一个数据节点后,就已经成为一个工作的系统。添加新的节点进集群时,有两个步骤,第一步:使用TDengine CLI连接到现有工作的数据节点,然后用命令”create dnode"将新的数据节点的End Point添加进去; 第二步:在新的数据节点的系统配置参数文件taos.cfg里,将first, second参数设置为现有集群中任意两个数据节点的EP即可。具体添加的详细步骤请见详细的用户手册。这样就把集群一步一步的建立起来。 +**新数据节点的加入**:系统有了一个数据节点后,就已经成为一个工作的系统。添加新的节点进集群时,有两个步骤,第一步:使用TDengine CLI连接到现有工作的数据节点,然后用命令”create dnode"将新的数据节点的End Point添加进去; 第二步:在新的数据节点的系统配置参数文件taos.cfg里,将firstEp, secondEp参数设置为现有集群中任意两个数据节点的EP即可。具体添加的详细步骤请见详细的用户手册。这样就把集群一步一步的建立起来。 **重定向**:无论是dnode还是taosc,最先都是要发起与mnode的连接,但mnode是系统自动创建并维护的,因此对于用户来说,并不知道哪个dnode在运行mnode。TDengine只要求向系统中任何一个工作的dnode发起连接即可。因为任何一个正在运行的dnode,都维护有目前运行的mnode EP List。当收到一个来自新启动的dnode或taosc的连接请求,如果自己不是mnode,则将mnode EP List回复给对方,taosc或新启动的dnode收到这个list, 就重新尝试建立连接。当mnode EP List发生改变,通过节点之间的消息交互,各个数据节点就很快获取最新列表,并通知taosc。 ### 一个典型的消息流程 + 为解释vnode, mnode, taosc和应用之间的关系以及各自扮演的角色,下面对写入数据这个典型操作的流程进行剖析。 -
    + +![TDengine典型的操作流程](page://images/architecture/message.png)
    图 2 TDengine典型的操作流程
    + 1. 应用通过JDBC、ODBC或其他API接口发起插入数据的请求。 2. taosc会检查缓存,看是否保存有该表的meta data。如果有,直接到第4步。如果没有,taosc将向mnode发出get meta-data请求。 3. mnode将该表的meta-data返回给taosc。Meta-data包含有该表的schema, 而且还有该表所属的vgroup信息(vnode ID以及所在的dnode的End Point,如果副本数为N,就有N组End Point)。如果taosc迟迟得不到mnode回应,而且存在多个mnode, taosc将向下一个mnode发出请求。 @@ -198,14 +212,15 @@ TDengine 分布式架构的逻辑结构图如下: 通过taosc缓存机制,只有在第一次对一张表操作时,才需要访问mnode,因此mnode不会成为系统瓶颈。但因为schema有可能变化,而且vgroup有可能发生改变(比如负载均衡发生),因此taosc会定时和mnode交互,自动更新缓存。 -## 存储模型与数据分区、分片 +## 存储模型与数据分区、分片 ### 存储模型 + TDengine存储的数据包括采集的时序数据以及库、表相关的元数据、标签数据等,这些数据具体分为三部分: -- 时序数据:存放于vnode里,由data、head和last三个文件组成,数据量大,查询量取决于应用场景。容许乱序写入,但暂时不支持删除和更新操作。通过采用一个采集点一张表的模型,一个时间段的数据是连续存储,对单张表的写入是简单的追加操作,一次读,可以读到多条记录,这样保证对单个采集点的插入和查询操作,性能达到最优。 +- 时序数据:存放于vnode里,由data、head和last三个文件组成,数据量大,查询量取决于应用场景。容许乱序写入,但暂时不支持删除操作,并且仅在update参数设置为1时允许更新操作。通过采用一个采集点一张表的模型,一个时间段的数据是连续存储,对单张表的写入是简单的追加操作,一次读,可以读到多条记录,这样保证对单个采集点的插入和查询操作,性能达到最优。 - 标签数据:存放于vnode里的meta文件,支持增删改查四个标准操作。数据量不大,有N张表,就有N条记录,因此可以全内存存储。如果标签过滤操作很多,查询将十分频繁,因此TDengine支持多核多线程并发查询。只要计算资源足够,即使有数千万张表,过滤结果能毫秒级返回。 -- 其他元数据:存放于mnode里,包含系统节点、用户、DB、Table Schema等等,支持增删改查四个标准操作。这部分数据的量不大,可以全内存保存,而且由于客户端有缓存,查询量也不大。因此目前的设计虽是集中式存储管理,但不会构成性能瓶颈。 +- 元数据:存放于mnode里,包含系统节点、用户、DB、Table Schema等信息,支持增删改查四个标准操作。这部分数据的量不大,可以全内存保存,而且由于客户端有缓存,查询量也不大。因此目前的设计虽是集中式存储管理,但不会构成性能瓶颈。 与典型的NoSQL存储模型相比,TDengine将标签数据与时序数据完全分离存储,它具有两大优势: @@ -213,6 +228,7 @@ TDengine存储的数据包括采集的时序数据以及库、表相关的元数 - 能够实现极为高效的多表之间的聚合查询:做多表之间聚合查询时,先把符合标签过滤条件的表查找出来,然后再查找这些表相应的数据块,这样大幅减少要扫描的数据集,从而大幅提高查询效率。而且标签数据采用全内存的结构进行管理和维护,千万级别规模的标签数据查询可以在毫秒级别返回。 ### 数据分片 + 对于海量的数据管理,为实现水平扩展,一般都需要采取分片(Sharding)分区(Partitioning)策略。TDengine是通过vnode来实现数据分片的,通过一个时间段一个数据文件来实现时序数据分区的。 vnode(虚拟数据节点)负责为采集的时序数据提供写入、查询和计算功能。为便于负载均衡、数据恢复、支持异构环境,TDengine将一个数据节点根据其计算和存储资源切分为多个vnode。这些vnode的管理是TDengine自动完成的,对应用完全透明。 @@ -224,11 +240,13 @@ vnode(虚拟数据节点)负责为采集的时序数据提供写入、查询和 每张表的meda data(包含schema, 标签等)也存放于vnode里,而不是集中存放于mnode,实际上这是对Meta数据的分片,这样便于高效并行的进行标签过滤操作。 ### 数据分区 + TDengine除vnode分片之外,还对时序数据按照时间段进行分区。每个数据文件只包含一个时间段的时序数据,时间段的长度由DB的配置参数days决定。这种按时间段分区的方法还便于高效实现数据的保留策略,只要数据文件超过规定的天数(系统配置参数keep),将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质,以便于大数据的冷热管理,实现多级存储。 总的来说,**TDengine是通过vnode以及时间两个维度,对大数据进行切分**,便于并行高效的管理,实现水平扩展。 ### 负载均衡 + 每个dnode都定时向 mnode(虚拟管理节点)报告其状态(包括硬盘空间、内存大小、CPU、网络、虚拟节点个数等),因此mnode了解整个集群的状态。基于整体状态,当mnode发现某个dnode负载过重,它会将dnode上的一个或多个vnode挪到其他dnode。在挪动过程中,对外服务继续进行,数据插入、查询和计算操作都不受影响。 如果mnode一段时间没有收到dnode的状态报告,mnode会认为这个dnode已经离线。如果离线时间超过一定时长(时长由配置参数offlineThreshold决定),该dnode将被mnode强制剔除出集群。该dnode上的vnodes如果副本数大于一,系统将自动在其他dnode上创建新的副本,以保证数据的副本数。如果该dnode上还有mnode, 而且mnode的副本数大于一,系统也将自动在其他dnode上创建新的mnode, 以保证mnode的副本数。 @@ -238,25 +256,31 @@ TDengine除vnode分片之外,还对时序数据按照时间段进行分区。 负载均衡过程无需任何人工干预,应用也无需重启,将自动连接新的节点,完全透明。 **提示:负载均衡由参数balance控制,决定开启/关闭自动负载均衡。** -## 数据写入与复制流程 +## 数据写入与复制流程 + 如果一个数据库有N个副本,那一个虚拟节点组就有N个虚拟节点,但是只有一个是Master,其他都是slave。当应用将新的记录写入系统时,只有Master vnode能接受写的请求。如果slave vnode收到写的请求,系统将通知taosc需要重新定向。 + ### Master vnode写入流程 + Master Vnode遵循下面的写入流程: -
    +![TDengine Master写入流程](page://images/architecture/write_master.png)
    图 3 TDengine Master写入流程
    + 1. Master vnode收到应用的数据插入请求,验证OK,进入下一步; 2. 如果系统配置参数walLevel大于0,vnode将把该请求的原始数据包写入数据库日志文件WAL。如果walLevel设置为2,而且fsync设置为0,TDengine还将WAL数据立即落盘,以保证即使宕机,也能从数据库日志文件中恢复数据,避免数据的丢失; 3. 如果有多个副本,vnode将把数据包转发给同一虚拟节点组内slave vnodes, 该转发包带有数据的版本号(version); -4. 写入内存,并加记录加入到skip list; +4. 写入内存,并将记录加入到skip list; 5. Master vnode返回确认信息给应用,表示写入成功。 6. 如果第2,3,4步中任何一步失败,将直接返回错误给应用。 ### Slave vnode写入流程 + 对于slave vnode, 写入流程是: -
    +![TDengine Slave写入流程](page://images/architecture/write_slave.png)
    图 4 TDengine Slave写入流程
    + 1. Slave vnode收到Master vnode转发了的数据插入请求。 2. 如果系统配置参数walLevel大于0,vnode将把该请求的原始数据包写入数据库日志文件WAL。如果walLevel设置为2,而且fsync设置为0,TDengine还将WAL数据立即落盘,以保证即使宕机,也能从数据库日志文件中恢复数据,避免数据的丢失; 3. 写入内存,更新内存中的skip list。 @@ -264,6 +288,7 @@ Master Vnode遵循下面的写入流程: 与Master vnode相比,slave vnode不存在转发环节,也不存在回复确认环节,少了两步。但写内存与WAL是完全一样的。 ### 异地容灾、IDC迁移 + 从上述Master和Slave流程可以看出,TDengine采用的是异步复制的方式进行数据同步。这种方式能够大幅提高写入性能,网络延时对写入速度不会有大的影响。通过配置每个物理节点的IDC和机架号,可以保证对于一个虚拟节点组,虚拟节点由来自不同IDC、不同机架的物理节点组成,从而实现异地容灾。因此TDengine原生支持异地容灾,无需再使用其他工具。 另外一方面,TDengine支持动态修改副本数,一旦副本数增加,新加入的虚拟节点将立即进入数据同步流程,同步结束后,新加入的虚拟节点即可提供服务。而在同步过程中,master以及其他已经同步的虚拟节点都可以对外提供服务。利用这一特性,TDengine可以实现无服务中断的IDC机房迁移。只需要将新IDC的物理节点加入现有集群,等数据同步完成后,再将老的IDC的物理节点从集群中剔除即可。 @@ -280,6 +305,7 @@ Master Vnode遵循下面的写入流程: **提示:该功能暂未提供** ### 主从选择 + Vnode会保持一个数据版本号(Version),对内存数据进行持久化存储时,对该版本号也进行持久化存储。每个数据更新操作,无论是采集的时序数据还是元数据,这个版本号将增一。 一个vnode启动时,角色(master、slave) 是不定的,数据是处于未同步状态,它需要与虚拟节点组内其他节点建立TCP连接,并互相交换status,其中包括version和自己的角色。通过status的交换,系统进入选主流程,规则如下: @@ -289,17 +315,20 @@ Vnode会保持一个数据版本号(Version),对内存数据进行持久化存 3. 在线的虚拟节点数过半,而且有虚拟节点是slave的话,该虚拟节点自动成为master 4. 对于2和3,如果多个虚拟节点满足成为master的要求,那么虚拟节点组的节点列表里,最前面的选为master -更多的关于数据复制的流程,请见TDengine 2.0数据复制模块设计。 +更多的关于数据复制的流程,请见[TDengine 2.0数据复制模块设计](https://www.taosdata.com/cn/documentation/architecture/replica/)。 ### 同步复制 + 对于数据一致性要求更高的场景,异步数据复制无法满足要求,因为有极小的概率丢失数据,因此TDengine提供同步复制的机制供用户选择。在创建数据库时,除指定副本数replica之外,用户还需要指定新的参数quorum。如果quorum大于一,它表示每次Master转发给副本时,需要等待quorum-1个回复确认,才能通知应用,数据在slave已经写入成功。如果在一定的时间内,得不到quorum-1个回复确认,master vnode将返回错误给应用。 采用同步复制,系统的性能会有所下降,而且latency会增加。因为元数据要强一致,mnode之间的数据同步缺省就是采用的同步复制。 注:vnode之间的同步复制仅仅企业版支持 -## 缓存与持久化 +## 缓存与持久化 + ### 缓存 + TDengine采用时间驱动缓存管理策略(First-In-First-Out,FIFO),又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式(Least-Recent-Used,LRU),直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候,将最早的数据批量写入磁盘。一般意义上来说,对于物联网数据的使用,用户最为关心的是刚产生的数据,即当前状态。TDengine充分利用这一特性,将最近到达的(当前状态)数据保存在缓存中。 TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中,可以更加快速地响应用户针对最近一条或一批数据的查询分析,整体上提供更快的数据库查询响应能力。从这个意义上来说,**可通过设置合适的配置参数将TDengine作为数据缓存来使用,而不需要再部署Redis或其他额外的缓存系统**,可有效地简化系统架构,降低运维的成本。需要注意的是,TDengine重启以后系统的缓存将被清空,之前缓存的数据均会被批量写入磁盘,缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。 @@ -307,6 +336,7 @@ TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接 每个vnode有自己独立的内存,而且由多个固定大小的内存块组成,不同vnode之间完全隔离。数据写入时,类似于日志的写法,数据被顺序追加写入内存,但每个vnode维护有自己的skip list,便于迅速查找。当三分之一以上的内存块写满时,启动落盘操作,而且后续写的操作在新的内存块进行。这样,一个vnode里有三分之一内存块是保留有最近的数据的,以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定,内存块的大小由配置参数cache决定。 ### 持久化存储 + TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时,为了不阻塞后续数据的写入,TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件,在落盘成功后则会删除老的数据库日志文件,避免日志文件无限制的增长。 为充分利用时序数据特点,TDengine将一个vnode保存在持久化存储的数据切分成多个文件,每个文件只保存固定天数的数据,这个天数由系统配置参数days决定。切分成多个文件后,给定查询的起止日期,无需任何索引,就可以立即定位需要打开哪些数据文件,大大加快读取速度。 @@ -322,6 +352,7 @@ TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久 数据写入磁盘时,根据系统配置参数comp决定是否压缩数据。TDengine提供了三种压缩选项:无压缩、一阶段压缩和两阶段压缩,分别对应comp值为0、1和2的情况。一阶段压缩根据数据的类型进行了相应的压缩,压缩算法包括delta-delta编码、simple 8B方法、zig-zag编码、LZ4等算法。二阶段压缩在一阶段压缩的基础上又用通用压缩算法进行了压缩,压缩率更高。 ### 多级存储 + 在默认配置下,TDengine会将所有数据保存在/var/lib/taos目录下,而且每个vnode的数据文件保存在该目录下的不同目录。为扩大存储空间,尽量减少文件读取的瓶颈,提高数据吞吐率 TDengine可通过配置系统参数dataDir让多个挂载的硬盘被系统同时使用。除此之外,TDengine也提供了数据分级存储的功能,即根据数据文件的新老程度存储在不同的存储介质上。比如最新的数据存储在SSD上,超过一周的数据存储在本地硬盘上,超过4周的数据存储在网络存储设备上,这样来降低存储成本,而又保证高效的访问数据。数据在不同存储介质上的移动是由系统自动完成的,对应用是完全透明的。数据的分级存储也是通过系统参数dataDir来配置。 dataDir的配置格式如下: @@ -344,12 +375,13 @@ dataDir /mnt/disk6/taos 2 挂载的盘也可以是非本地的网络盘,只要系统能访问即可。 注:多级存储功能仅企业版支持 -**提示:该功能暂未提供** -## 数据查询 +## 数据查询 + TDengine提供了多种多样针对表和超级表的查询处理功能,除了常规的聚合查询之外,还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、vnode, mnode节点协同完成。 ### 单表查询 + SQL语句的解析和校验工作在客户端完成。解析SQL语句并生成抽象语法树(Abstract Syntax Tree, AST),然后对其进行校验和检查。以及向管理节点(mnode)请求查询中指定表的元数据信息(table metadata)。 根据元数据信息中的End Point信息,将查询请求序列化后发送到该表所在的数据节点(dnode)。dnode接收到查询请求后,识别出该查询请求指向的虚拟节点(vnode),将消息转发到vnode的查询执行队列。vnode的查询执行线程建立基础的查询执行环境,并立即返回该查询请求,同时开始执行该查询。 @@ -372,21 +404,24 @@ select count(*) from d1001 interval(1h); select count(*) from d1001 interval(1h) fill(prev); ``` -针对d1001设备采集数据统计每小时记录数,如果某一个小时不存在数据,这返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。 +针对d1001设备采集数据统计每小时记录数,如果某一个小时不存在数据,则返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。 ### 多表聚合查询 + TDengine对每个数据采集点单独建表,但在实际应用中经常需要对不同的采集点数据进行聚合。为高效的进行聚合操作,TDengine引入超级表(STable)的概念。超级表用来代表一特定类型的数据采集点,它是包含多张表的表集合,集合里每张表的模式(schema)完全一致,但每张表都带有自己的静态标签,标签可以多个,可以随时增加、删除和修改。 应用可通过指定标签的过滤条件,对一个STable下的全部或部分表进行聚合或统计操作,这样大大简化应用的开发。其具体流程如下图所示: -
    +![多表聚合查询原理图](page://images/architecture/multi_tables.png)
    图 5 多表聚合查询原理图
    -1:应用将一个查询条件发往系统; -2: taosc将超级表的名字发往 Meta Node(管理节点); -3:管理节点将超级表所拥有的 vnode 列表发回 taosc; -4:taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点; -5:每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合,然后扫描存储的时序数据,完成相应的聚合计算,将结果返回给taosc; -6:taosc将多个数据节点返回的结果做最后的聚合,将其返回给应用。 + +1. 应用将一个查询条件发往系统; +2. taosc将超级表的名字发往 Meta Node(管理节点); +3. 管理节点将超级表所拥有的 vnode 列表发回 taosc; +4. taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点; +5. 每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合,然后扫描存储的时序数据,完成相应的聚合计算,将结果返回给taosc; +6. taosc将多个数据节点返回的结果做最后的聚合,将其返回给应用。 由于TDengine在vnode内将标签数据与时序数据分离存储,通过在内存里过滤标签数据,先找到需要参与聚合操作的表的集合,将需要扫描的数据集大幅减少,大幅提升聚合计算速度。同时,由于数据分布在多个vnode/dnode,聚合计算操作在多个vnode里并发进行,又进一步提升了聚合的速度。 对普通表的聚合函数以及绝大部分操作都适用于超级表,语法完全一样,细节请看 TAOS SQL。 ### 预计算 + 为有效提升查询处理的性能,针对物联网数据的不可更改的特点,在数据块头部记录该数据块中存储数据的统计信息:包括最大值、最小值、和。我们称之为预计算单元。如果查询处理涉及整个数据块的全部数据,直接使用预计算结果,完全不需要读取数据块的内容。由于预计算数据量远小于磁盘上存储的数据块数据的大小,对于磁盘IO为瓶颈的查询处理,使用预计算结果可以极大地减小读取IO压力,加速查询处理的流程。预计算机制与Postgre SQL的索引BRIN(block range index)有异曲同工之妙。 diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/cn/04.model/docs.md similarity index 81% rename from documentation20/webdocs/markdowndocs/Model-ch.md rename to documentation20/cn/04.model/docs.md index ea1be899a85fe6bb31ab03674ab496d7b301432f..6f85381588c6cd131f2204bba13e64e7058f93f2 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/cn/04.model/docs.md @@ -4,16 +4,16 @@ TDengine采用关系型数据模型,需要建库、建表。因此对于一个具体的应用场景,需要考虑库的设计,超级表和普通表的设计。本节不讨论细致的语法规则,只介绍概念。 -关于数据建模请参考视频教程。 +关于数据建模请参考[视频教程](https://www.taosdata.com/blog/2020/11/11/1945.html)。 -## 创建库 +## 创建库 不同类型的数据采集点往往具有不同的数据特征,包括数据采集频率的高低,数据保留时间的长短,副本的数目,数据块的大小,是否允许更新数据等等。为让各种场景下TDengine都能最大效率的工作,TDengine建议将不同数据特征的表创建在不同的库里,因为每个库可以配置不同的存储策略。创建一个库时,除SQL标准的选项外,应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如: ```mysql CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4 UPDATE 1; ``` -上述语句将创建一个名为power的库,这个库的数据将保留365天(超过365天将被自动删除),每10天一个数据文件,内存块数为4,允许更新数据。详细的语法及参数请见TAOS SQL +上述语句将创建一个名为power的库,这个库的数据将保留365天(超过365天将被自动删除),每10天一个数据文件,内存块数为4,允许更新数据。详细的语法及参数请见 [TAOS SQL 的数据管理](https://www.taosdata.com/cn/documentation/taos-sql#management) 章节。 创建库之后,需要使用SQL命令USE将当前库切换过来,例如: @@ -28,23 +28,31 @@ USE power; - 任何一张表或超级表是属于一个库的,在创建表之前,必须先创建库。 - 处于两个不同库的表是不能进行JOIN操作的。 -## 创建超级表 +## 创建超级表 + 一个物联网系统,往往存在多种类型的设备,比如对于电网,存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合,使用TDengine, 需要对每个类型的数据采集点创建一超级表。以表一中的智能电表为例,可以使用如下的SQL命令创建超级表: + ```mysql -CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); +CREATE STABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); ``` -与创建普通表一样,创建表时,需要提供表名(示例中为meters),表结构Schema,即数据列的定义。第一列必须为时间戳(示例中为ts),其他列为采集的物理量(示例中为current, voltage, phase),数据类型可以为整型、浮点型、字符串等。除此之外,还需要提供标签的schema (示例中为location, groupId),标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签,比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 + +**注意:**这一指令中的 STABLE 关键字,在 2.0.15 之前的版本中需写作 TABLE 。 + +与创建普通表一样,创建表时,需要提供表名(示例中为meters),表结构Schema,即数据列的定义。第一列必须为时间戳(示例中为ts),其他列为采集的物理量(示例中为current, voltage, phase),数据类型可以为整型、浮点型、字符串等。除此之外,还需要提供标签的schema (示例中为location, groupId),标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签,比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 [TAOS SQL 的超级表管理](https://www.taosdata.com/cn/documentation/taos-sql#super-table) 章节。 每一种类型的数据采集点需要建立一个超级表,因此一个物联网系统,往往会有多个超级表。对于电网,我们就需要对智能电表、变压器、母线、开关等都建立一个超级表。在物联网中,一个设备就可能有多个数据采集点(比如一台风力发电的风机,有的采集点采集电流、电压等电参数,有的采集点采集温度、湿度、风向等环境参数),这个时候,对这一类型的设备,需要建立多张超级表。一张超级表里包含的采集物理量必须是同时采集的(时间戳是一致的)。 一张超级表最多容许1024列,如果一个采集点采集的物理量个数超过1024,需要建多张超级表来处理。一个系统可以有多个DB,一个DB里可以有一到多个超级表。 -## 创建表 +## 创建表 + TDengine对每个数据采集点需要独立建表。与标准的关系型数据一样,一张表有表名,Schema,但除此之外,还可以带有一到多个标签。创建时,需要使用超级表做模板,同时指定标签的具体值。以表一中的智能电表为例,可以使用如下的SQL命令建表: -```cmd + +```mysql CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2); ``` -其中d1001是表名,meters是超级表的表名,后面紧跟标签Location的具体标签值”Beijing.Chaoyang",标签groupId的具体标签值2。虽然在创建表时,需要指定标签值,但可以事后修改。详细细则请见 TAOS SQL。 + +其中d1001是表名,meters是超级表的表名,后面紧跟标签Location的具体标签值”Beijing.Chaoyang",标签groupId的具体标签值2。虽然在创建表时,需要指定标签值,但可以事后修改。详细细则请见 [TAOS SQL 的表管理](https://www.taosdata.com/cn/documentation/taos-sql#table) 章节。 **注意:**目前 TDengine 没有从技术层面限制使用一个 database (dbA)的超级表作为模板建立另一个 database (dbB)的子表,后续会禁止这种用法,不建议使用这种方法建表。 @@ -52,12 +60,16 @@ TDengine建议将数据采集点的全局唯一ID作为表名(比如设备序列 **自动建表**:在某些特殊场景中,用户在写数据时并不确定某个数据采集点的表是否存在,此时可在写入数据时使用自动建表语法来创建不存在的表,若该表已存在则不会建立新表。比如: -```cmd +```mysql INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32); ``` -上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建,则使用超级表meters做模板自动创建,同时打上标签值“Beijing.Chaoyang", 2。 + +上述SQL语句将记录 (now, 10.2, 219, 0.32) 插入表d1001。如果表d1001还未创建,则使用超级表meters做模板自动创建,同时打上标签值“Beijing.Chaoyang", 2。 + +关于自动建表的详细语法请参见 [插入记录时自动建表](https://www.taosdata.com/cn/documentation/taos-sql#auto_create_table) 章节。 ## 多列模型 vs 单列模型 + TDengine支持多列模型,只要物理量是一个数据采集点同时采集的(时间戳一致),这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计,单列模型,每个采集的物理量都单独建表,因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位,就建三张超级表。 TDengine建议尽可能采用多列模型,因为插入效率以及存储效率更高。但对于有些场景,一个采集点的采集量的种类经常变化,这个时候,如果采用多列模型,就需要频繁修改超级表的结构定义,让应用变的复杂,这个时候,采用单列模型会显得简单。 diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/cn/05.insert/docs.md similarity index 72% rename from documentation20/webdocs/markdowndocs/insert-ch.md rename to documentation20/cn/05.insert/docs.md index 3fa48c1f508bedf7b4000ffe9f7ef8c96e42d606..ce2d65e7d2259c6dac9efc67a61f7c009dd96984 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/cn/05.insert/docs.md @@ -2,7 +2,7 @@ TDengine支持多种接口写入数据,包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, HiveMQ Broker, CSV文件等,后续还将提供Kafka, OPC等接口。数据可以单条插入,也可以批量插入,可以插入一个数据采集点的数据,也可以同时插入多个数据采集点的数据。支持多线程插入,支持时间乱序数据插入,也支持历史数据插入。 -## SQL写入 +## SQL写入 应用通过C/C++, JDBC, GO, 或Python Connector 执行SQL insert语句来插入数据,用户还可以通过TAOS Shell,手动输入SQL insert语句插入数据。比如下面这条insert 就将一条记录写入到表d1001中: ```mysql @@ -18,7 +18,7 @@ TDengine也支持一次向多个表写入数据,比如下面这条命令就向 INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) d1002 VALUES (1538548696800, 12.3, 221, 0.31); ``` -详细的SQL INSERT语法规则请见TAOS SQL +详细的SQL INSERT语法规则请见 [TAOS SQL 的数据写入](https://www.taosdata.com/cn/documentation/taos-sql#insert) 章节。 **Tips:** @@ -27,11 +27,13 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, - 对同一张表,如果新插入记录的时间戳已经存在,默认(没有使用 UPDATE 1 创建数据库)新记录将被直接抛弃,也就是说,在一张表里,时间戳必须是唯一的。如果应用自动生成记录,很有可能生成的时间戳是一样的,这样,成功插入的记录条数会小于应用插入的记录条数。如果在创建数据库时使用 UPDATE 1 选项,插入相同时间戳的新记录将覆盖原有记录。 - 写入的数据的时间戳必须大于当前时间减去配置参数keep的时间。如果keep配置为3650天,那么无法写入比3650天还老的数据。写入数据的时间戳也不能大于当前时间加配置参数days。如果days配置为2,那么无法写入比当前时间还晚2天的数据。 -## Prometheus直接写入 -Prometheus作为Cloud Native Computing Fundation毕业的项目,在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具Bailongma,只需在Prometheus做简单配置,无需任何代码,就可将Prometheus采集的数据直接写入TDengine,并按规则在TDengine自动创建库和相关表项。博文用Docker容器快速搭建一个Devops监控Demo即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例,可以参考。 +## Prometheus直接写入 + +[Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目,在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma),只需在Prometheus做简单配置,无需任何代码,就可将Prometheus采集的数据直接写入TDengine,并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例,可以参考。 ### 从源代码编译blm_prometheus -用户需要从github下载Bailongma的源码,使用Golang语言编译器编译生成可执行文件。在开始编译前,需要准备好以下条件: + +用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码,使用Golang语言编译器编译生成可执行文件。在开始编译前,需要准备好以下条件: - Linux操作系统的服务器 - 安装好Golang, 1.10版本以上 - 对应的TDengine版本。因为用到了TDengine的客户端动态链接库,因此需要安装好和服务端相同版本的TDengine程序;比如服务端版本是TDengine 2.0.0, 则在bailongma所在的linux服务器(可以与TDengine在同一台服务器,或者不同服务器) @@ -45,10 +47,12 @@ go build 一切正常的情况下,就会在对应的目录下生成一个blm_prometheus的可执行程序。 ### 安装Prometheus -通过Prometheus的官网下载安装。下载地址 + +通过Prometheus的官网下载安装。[下载地址](https://prometheus.io/download/) ### 配置Prometheus -参考Prometheus的配置文档,在Prometheus的配置文件中的部分,增加以下配置 + +参考Prometheus的[配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/),在Prometheus的配置文件中的部分,增加以下配置 - url: bailongma API服务提供的URL, 参考下面的blm_prometheus启动示例章节 @@ -112,11 +116,13 @@ use prometheus; select * from apiserver_request_latencies_bucket; ``` -## Telegraf直接写入 -是一流行的IT运维数据采集开源工具,TDengine提供一个小工具Bailongma,只需在Telegraf做简单配置,无需任何代码,就可将Telegraf采集的数据直接写入TDengine,并按规则在TDengine自动创建库和相关表项。博文用Docker容器快速搭建一个Devops监控Demo即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例,可以参考。 +## Telegraf直接写入 + +[Telegraf](https://www.influxdata.com/time-series-platform/telegraf/)是一流行的IT运维数据采集开源工具,TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma),只需在Telegraf做简单配置,无需任何代码,就可将Telegraf采集的数据直接写入TDengine,并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例,可以参考。 ### 从源代码编译blm_telegraf -用户需要从github下载Bailongma的源码,使用Golang语言编译器编译生成可执行文件。在开始编译前,需要准备好以下条件: + +用户需要从github下载[Bailongma](https://github.com/taosdata/Bailongma)的源码,使用Golang语言编译器编译生成可执行文件。在开始编译前,需要准备好以下条件: - Linux操作系统的服务器 - 安装好Golang, 1.10版本以上 @@ -132,9 +138,11 @@ go build 一切正常的情况下,就会在对应的目录下生成一个blm_telegraf的可执行程序。 ### 安装Telegraf -目前TDengine支持Telegraf 1.7.4以上的版本。用户可以根据当前的操作系统,到Telegraf官网下载安装包,并执行安装。下载地址如下:https://portal.influxdata.com/downloads + +目前TDengine支持Telegraf 1.7.4以上的版本。用户可以根据当前的操作系统,到Telegraf官网下载安装包,并执行安装。下载地址如下:https://portal.influxdata.com/downloads ### 配置Telegraf + 修改Telegraf配置文件/etc/telegraf/telegraf.conf中与TDengine有关的配置项。 在output plugins部分,增加[[outputs.http]]配置项: @@ -148,7 +156,7 @@ go build - hostname: 区分不同采集设备的机器名称,需确保其唯一性 - metric_batch_size: 100,允许Telegraf每批次写入记录最大数量,增大其数量可以降低Telegraf的请求发送频率。 -关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息,请参考Telegraf官方的文档。 +关于如何使用Telegraf采集数据以及更多有关使用Telegraf的信息,请参考Telegraf官方的[文档](https://docs.influxdata.com/telegraf/v1.11/)。 ### 启动blm_telegraf程序 blm_telegraf程序有以下选项,在启动blm_telegraf程序时可以通过设定这些选项来设定blm_telegraf的配置。 @@ -174,6 +182,7 @@ blm_telegraf对telegraf提供服务的端口号。 ``` ### 启动示例 + 通过以下命令启动一个blm_telegraf的API服务 ```bash ./blm_telegraf -host 127.0.0.1 -port 8089 @@ -186,6 +195,7 @@ url = "http://10.1.2.3:8089/telegraf" ``` ### 查询telegraf写入数据 + telegraf产生的数据格式如下: ```json { @@ -220,10 +230,10 @@ select * from cpu; MQTT是一流行的物联网数据传输协议,TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。 -## EMQ Broker 直接写入 +## EMQ Broker 直接写入 -EMQ是一开源的MQTT Broker软件,无需任何代码,只需要在EMQ Dashboard里使用“规则”做简单配置,即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine,也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考EMQ 官方文档。 +[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件,无需任何代码,只需要在EMQ Dashboard里使用“规则”做简单配置,即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine,也在企业版上提供原生的 TDengine 驱动实现直接保存。详细使用方法请参考[EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 -## HiveMQ Broker 直接写入 +## HiveMQ Broker 直接写入 -HiveMQ 是一个提供免费个人版和企业版的 MQTT 代理,主要用于企业和新兴的机器到机器M2M通讯和内部传输,满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 HiveMQ extension - TDengine 说明文档。 +[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理,主要用于企业和新兴的机器到机器M2M通讯和内部传输,满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。 diff --git a/documentation20/webdocs/markdowndocs/Queries-ch.md b/documentation20/cn/06.queries/docs.md similarity index 89% rename from documentation20/webdocs/markdowndocs/Queries-ch.md rename to documentation20/cn/06.queries/docs.md index 960bb39e6323fe590905de62d2a1021adb19d176..a161778a72728ca05a75538c8b04ca0277e01bb2 100644 --- a/documentation20/webdocs/markdowndocs/Queries-ch.md +++ b/documentation20/cn/06.queries/docs.md @@ -1,15 +1,12 @@ - - - # 高效查询数据 -## 主要查询功能 +## 主要查询功能 TDengine 采用 SQL 作为查询语言。应用程序可以通过 C/C++, Java, Go, Python 连接器发送 SQL 语句,用户可以通过 TDengine 提供的命令行(Command Line Interface, CLI)工具 TAOS Shell 手动执行 SQL 即席查询(Ad-Hoc Query)。TDengine 支持如下查询功能: - 单列、多列数据查询 -- 标签和数值的多种过滤条件:\>, \<, =, \<>, like 等 +- 标签和数值的多种过滤条件:>, <, =, <>, like 等 - 聚合结果的分组(Group by)、排序(Order by)、约束输出(Limit/Offset) - 数值列及聚合结果的四则运算 - 时间戳对齐的连接查询(Join Query: 隐式连接)操作 @@ -26,9 +23,10 @@ Query OK, 2 row(s) in set (0.001100s) ``` 为满足物联网场景的需求,TDengine支持几个特殊的函数,比如twa(时间加权平均),spread (最大值与最小值的差),last_row(最后一条记录)等,更多与物联网场景相关的函数将添加进来。TDengine还支持连续查询。 -具体的查询语法请看TAOS SQL 。 +具体的查询语法请看 [TAOS SQL 的数据查询](https://www.taosdata.com/cn/documentation/taos-sql#select) 章节。 + +## 多表聚合查询 -## 多表聚合查询 物联网场景中,往往同一个类型的数据采集点有多个。TDengine采用超级表(STable)的概念来描述某一个类型的数据采集点,一张普通的表来描述一个具体的数据采集点。同时TDengine使用标签来描述数据采集点的静态属性,一个具体的数据采集点有具体的标签值。通过指定标签的过滤条件,TDengine提供了一高效的方法将超级表(某一类型的数据采集点)所属的子表进行聚合查询。对普通表的聚合函数以及绝大部分操作都适用于超级表,语法完全一样。 **示例1**:在TAOS Shell,查找北京所有智能电表采集的电压平均值,并按照location分组 @@ -51,9 +49,9 @@ taos> SELECT count(*), max(current) FROM meters where groupId = 2 and ts > now - Query OK, 1 row(s) in set (0.002136s) ``` -TDengine仅容许对属于同一个超级表的表之间进行聚合查询,不同超级表之间的聚合查询不支持。在TAOS SQL 一章,查询类操作都会注明是否支持超级表。 +TDengine仅容许对属于同一个超级表的表之间进行聚合查询,不同超级表之间的聚合查询不支持。在 [TAOS SQL 的数据查询](https://www.taosdata.com/cn/documentation/taos-sql#select) 一章,查询类操作都会注明是否支持超级表。 -## 降采样查询、插值 +## 降采样查询、插值 物联网场景里,经常需要通过降采样(down sampling)将采集的数据按时间段进行聚合。TDengine 提供了一个简便的关键词 interval 让按照时间窗口的查询操作变得极为简单。比如,将智能电表 d1001 采集的电流值每10秒钟求和 ```mysql @@ -91,5 +89,5 @@ Query OK, 5 row(s) in set (0.001521s) 物联网场景里,每个数据采集点采集数据的时间是难同步的,但很多分析算法(比如FFT)需要把采集的数据严格按照时间等间隔的对齐,在很多系统里,需要应用自己写程序来处理,但使用TDengine的降采样操作就轻松解决。如果一个时间间隔里,没有采集的数据,TDengine还提供插值计算的功能。 -语法规则细节请见TAOS SQL 。 +语法规则细节请见 [TAOS SQL 的时间维度聚合](https://www.taosdata.com/cn/documentation/taos-sql#aggregation) 章节。 diff --git a/documentation20/webdocs/markdowndocs/advanced features-ch.md b/documentation20/cn/07.advanced-features/docs.md similarity index 64% rename from documentation20/webdocs/markdowndocs/advanced features-ch.md rename to documentation20/cn/07.advanced-features/docs.md index cdd9ee81048f0968fb02f036b20e003c66835a4c..650a2ca96b759bd6b8123dbb64023496b654dcd0 100644 --- a/documentation20/webdocs/markdowndocs/advanced features-ch.md +++ b/documentation20/cn/07.advanced-features/docs.md @@ -1,29 +1,16 @@ # 高级功能 -## 连续查询(Continuous Query) +## 连续查询(Continuous Query) -连续查询是TDengine定期自动执行的查询,采用滑动窗口的方式进行计算,是一种简化的时间驱动的流式计算。 -针对库中的表或超级表,TDengine可提供定期自动执行的连续查询, -用户可让TDengine推送查询的结果,也可以将结果再写回到TDengine中。 -每次执行的查询是一个时间窗口,时间窗口随着时间流动向前滑动。 -在定义连续查询的时候需要指定时间窗口(time window, 参数interval)大小和每次前向增量时间(forward sliding times, 参数sliding)。 +连续查询是TDengine定期自动执行的查询,采用滑动窗口的方式进行计算,是一种简化的时间驱动的流式计算。针对库中的表或超级表,TDengine可提供定期自动执行的连续查询,用户可让TDengine推送查询的结果,也可以将结果再写回到TDengine中。每次执行的查询是一个时间窗口,时间窗口随着时间流动向前滑动。在定义连续查询的时候需要指定时间窗口(time window, 参数interval)大小和每次前向增量时间(forward sliding times, 参数sliding)。 -TDengine的连续查询采用时间驱动模式,可以直接使用TAOS SQL进行定义,不需要额外的操作。 -使用连续查询,可以方便快捷地按照时间窗口生成结果,从而对原始采集数据进行降采样(down sampling)。 -用户通过TAOS SQL定义连续查询以后,TDengine自动在最后的一个完整的时间周期末端拉起查询, -并将计算获得的结果推送给用户或者写回TDengine。 +TDengine的连续查询采用时间驱动模式,可以直接使用TAOS SQL进行定义,不需要额外的操作。使用连续查询,可以方便快捷地按照时间窗口生成结果,从而对原始采集数据进行降采样(down sampling)。用户通过TAOS SQL定义连续查询以后,TDengine自动在最后的一个完整的时间周期末端拉起查询,并将计算获得的结果推送给用户或者写回TDengine。 TDengine提供的连续查询与普通流计算中的时间窗口计算具有以下区别: -- 不同于流计算的实时反馈计算结果,连续查询只在时间窗口关闭以后才开始计算。 -例如时间周期是1天,那么当天的结果只会在23:59:59以后才会生成。 - -- 如果有历史记录写入到已经计算完成的时间区间,连续查询并不会重新进行计算, -也不会重新将结果推送给用户。对于写回TDengine的模式,也不会更新已经存在的计算结果。 - -- 使用连续查询推送结果的模式,服务端并不缓存客户端计算状态,也不提供Exactly-Once的语意保证。 -如果用户的应用端崩溃,再次拉起的连续查询将只会从再次拉起的时间开始重新计算最近的一个完整的时间窗口。 -如果使用写回模式,TDengine可确保数据写回的有效性和连续性。 +- 不同于流计算的实时反馈计算结果,连续查询只在时间窗口关闭以后才开始计算。例如时间周期是1天,那么当天的结果只会在23:59:59以后才会生成。 +- 如果有历史记录写入到已经计算完成的时间区间,连续查询并不会重新进行计算,也不会重新将结果推送给用户。对于写回TDengine的模式,也不会更新已经存在的计算结果。 +- 使用连续查询推送结果的模式,服务端并不缓存客户端计算状态,也不提供Exactly-Once的语意保证。如果用户的应用端崩溃,再次拉起的连续查询将只会从再次拉起的时间开始重新计算最近的一个完整的时间窗口。如果使用写回模式,TDengine可确保数据写回的有效性和连续性。 ### 使用连续查询 @@ -42,23 +29,19 @@ create table D1002 using meters tags ("Beijing.Haidian", 2); select avg(voltage) from meters interval(1m) sliding(30s); ``` -每次执行这条语句,都会重新计算所有数据。 -如果需要每隔30秒执行一次来增量计算最近一分钟的数据, -可以把上面的语句改进成下面的样子,每次使用不同的 `startTime` 并定期执行: +每次执行这条语句,都会重新计算所有数据。 如果需要每隔30秒执行一次来增量计算最近一分钟的数据,可以把上面的语句改进成下面的样子,每次使用不同的 `startTime` 并定期执行: ```sql select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s); ``` -这样做没有问题,但TDengine提供了更简单的方法, -只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如: +这样做没有问题,但TDengine提供了更简单的方法,只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如: ```sql create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s); ``` -会自动创建一个名为 `avg_vol` 的新表,然后每隔30秒,TDengine会增量执行 `as` 后面的 SQL 语句, -并将查询结果写入这个表中,用户程序后续只要从 `avg_vol` 中查询数据即可。 例如: +会自动创建一个名为 `avg_vol` 的新表,然后每隔30秒,TDengine会增量执行 `as` 后面的 SQL 语句,并将查询结果写入这个表中,用户程序后续只要从 `avg_vol` 中查询数据即可。 例如: ```mysql taos> select * from avg_vol; @@ -72,43 +55,27 @@ taos> select * from avg_vol; 需要注意,查询时间窗口的最小值是10毫秒,没有时间窗口范围的上限。 -此外,TDengine还支持用户指定连续查询的起止时间。 -如果不输入开始时间,连续查询将从第一条原始数据所在的时间窗口开始; -如果没有输入结束时间,连续查询将永久运行; -如果用户指定了结束时间,连续查询在系统时间达到指定的时间以后停止运行。 -比如使用下面的SQL创建的连续查询将运行一小时,之后会自动停止。 +此外,TDengine还支持用户指定连续查询的起止时间。如果不输入开始时间,连续查询将从第一条原始数据所在的时间窗口开始;如果没有输入结束时间,连续查询将永久运行;如果用户指定了结束时间,连续查询在系统时间达到指定的时间以后停止运行。比如使用下面的SQL创建的连续查询将运行一小时,之后会自动停止。 ```mysql create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s); ``` -需要说明的是,上面例子中的 `now` 是指创建连续查询的时间,而不是查询执行的时间,否则,查询就无法自动停止了。 -另外,为了尽量避免原始数据延迟写入导致的问题,TDengine中连续查询的计算有一定的延迟。 -也就是说,一个时间窗口过去后,TDengine并不会立即计算这个窗口的数据, -所以要稍等一会(一般不会超过1分钟)才能查到计算结果。 +需要说明的是,上面例子中的 `now` 是指创建连续查询的时间,而不是查询执行的时间,否则,查询就无法自动停止了。另外,为了尽量避免原始数据延迟写入导致的问题,TDengine中连续查询的计算有一定的延迟。也就是说,一个时间窗口过去后,TDengine并不会立即计算这个窗口的数据,所以要稍等一会(一般不会超过1分钟)才能查到计算结果。 ### 管理连续查询 -用户可在控制台中通过 `show streams` 命令来查看系统中全部运行的连续查询, -并可以通过 `kill stream` 命令杀掉对应的连续查询。 -后续版本会提供更细粒度和便捷的连续查询管理命令。 +用户可在控制台中通过 `show streams` 命令来查看系统中全部运行的连续查询,并可以通过 `kill stream` 命令杀掉对应的连续查询。后续版本会提供更细粒度和便捷的连续查询管理命令。 -## 数据订阅(Publisher/Subscriber) +## 数据订阅(Publisher/Subscriber) -基于数据天然的时间序列特性,TDengine的数据写入(insert)与消息系统的数据发布(pub)逻辑上一致, -均可视为系统中插入一条带时间戳的新记录。 -同时,TDengine在内部严格按照数据时间序列单调递增的方式保存数据。 -本质上来说,TDengine中里每一张表均可视为一个标准的消息队列。 +基于数据天然的时间序列特性,TDengine的数据写入(insert)与消息系统的数据发布(pub)逻辑上一致,均可视为系统中插入一条带时间戳的新记录。同时,TDengine在内部严格按照数据时间序列单调递增的方式保存数据。本质上来说,TDengine中里每一张表均可视为一个标准的消息队列。 -TDengine内嵌支持轻量级的消息订阅与推送服务。 -使用系统提供的API,用户可使用普通查询语句订阅数据库中的一张或多张表。 -订阅的逻辑和操作状态的维护均是由客户端完成,客户端定时轮询服务器是否有新的记录到达, -有新的记录到达就会将结果反馈到客户。 +TDengine内嵌支持轻量级的消息订阅与推送服务。使用系统提供的API,用户可使用普通查询语句订阅数据库中的一张或多张表。订阅的逻辑和操作状态的维护均是由客户端完成,客户端定时轮询服务器是否有新的记录到达,有新的记录到达就会将结果反馈到客户。 -TDengine的订阅与推送服务的状态是客户端维持,TDengine服务器并不维持。 -因此如果应用重启,从哪个时间点开始获取最新数据,由应用决定。 +TDengine的订阅与推送服务的状态是客户端维持,TDengine服务器并不维持。因此如果应用重启,从哪个时间点开始获取最新数据,由应用决定。 TDengine的API中,与订阅相关的主要有以下三个: @@ -118,12 +85,9 @@ taos_consume taos_unsubscribe ``` -这些API的文档请见 [C/C++ Connector](https://www.taosdata.com/cn/documentation20/connector/), -下面仍以智能电表场景为例介绍一下它们的具体用法(超级表和子表结构请参考上一节“连续查询”), -完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/tests/examples/c/subscribe.c) 找到。 +这些API的文档请见 [C/C++ Connector](https://www.taosdata.com/cn/documentation/connector#c-cpp),下面仍以智能电表场景为例介绍一下它们的具体用法(超级表和子表结构请参考上一节“连续查询”),完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/tests/examples/c/subscribe.c) 找到。 -如果我们希望当某个电表的电流超过一定限制(比如10A)后能得到通知并进行一些处理, 有两种方法: -一是分别对每张子表进行查询,每次查询后记录最后一条数据的时间戳,后续只查询这个时间戳之后的数据: +如果我们希望当某个电表的电流超过一定限制(比如10A)后能得到通知并进行一些处理, 有两种方法:一是分别对每张子表进行查询,每次查询后记录最后一条数据的时间戳,后续只查询这个时间戳之后的数据: ```sql select * from D1001 where ts > {last_timestamp1} and current > 10; @@ -131,8 +95,7 @@ select * from D1002 where ts > {last_timestamp2} and current > 10; ... ``` -这确实可行,但随着电表数量的增加,查询数量也会增加,客户端和服务端的性能都会受到影响, -当电表数增长到一定的程度,系统就无法承受了。 +这确实可行,但随着电表数量的增加,查询数量也会增加,客户端和服务端的性能都会受到影响,当电表数增长到一定的程度,系统就无法承受了。 另一种方法是对超级表进行查询。这样,无论有多少电表,都只需一次查询: @@ -140,12 +103,7 @@ select * from D1002 where ts > {last_timestamp2} and current > 10; select * from meters where ts > {last_timestamp} and current > 10; ``` -但是,如何选择 `last_timestamp` 就成了一个新的问题。 -因为,一方面数据的产生时间(也就是数据时间戳)和数据入库的时间一般并不相同,有时偏差还很大; -另一方面,不同电表的数据到达TDengine的时间也会有差异。 -所以,如果我们在查询中使用最慢的那台电表的数据的时间戳作为 `last_timestamp`, -就可能重复读入其它电表的数据; -如果使用最快的电表的时间戳,其它电表的数据就可能被漏掉。 +但是,如何选择 `last_timestamp` 就成了一个新的问题。因为,一方面数据的产生时间(也就是数据时间戳)和数据入库的时间一般并不相同,有时偏差还很大;另一方面,不同电表的数据到达TDengine的时间也会有差异。所以,如果我们在查询中使用最慢的那台电表的数据的时间戳作为 `last_timestamp`,就可能重复读入其它电表的数据;如果使用最快的电表的时间戳,其它电表的数据就可能被漏掉。 TDengine的订阅功能为上面这个问题提供了一个彻底的解决方案。 @@ -162,47 +120,29 @@ if (async) { } ``` -TDengine中的订阅既可以是同步的,也可以是异步的, -上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。 -这里,同步的意思是用户程序要直接调用`taos_consume`来拉取数据, -而异步则由API在内部的另一个线程中调用`taos_consume`, -然后把拉取到的数据交给回调函数`subscribe_callback`去处理。 +TDengine中的订阅既可以是同步的,也可以是异步的,上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。这里,同步的意思是用户程序要直接调用`taos_consume`来拉取数据,而异步则由API在内部的另一个线程中调用`taos_consume`,然后把拉取到的数据交给回调函数`subscribe_callback`去处理。 -参数`taos`是一个已经建立好的数据库连接,在同步模式下无特殊要求。 -但在异步模式下,需要注意它不会被其它线程使用,否则可能导致不可预计的错误, -因为回调函数在API的内部线程中被调用,而TDengine的部分API不是线程安全的。 +参数`taos`是一个已经建立好的数据库连接,在同步模式下无特殊要求。但在异步模式下,需要注意它不会被其它线程使用,否则可能导致不可预计的错误,因为回调函数在API的内部线程中被调用,而TDengine的部分API不是线程安全的。 -参数`sql`是查询语句,可以在其中使用where子句指定过滤条件。 -在我们的例子中,如果只想订阅电流超过10A时的数据,可以这样写: +参数`sql`是查询语句,可以在其中使用where子句指定过滤条件。在我们的例子中,如果只想订阅电流超过10A时的数据,可以这样写: ```sql select * from meters where current > 10; ``` -注意,这里没有指定起始时间,所以会读到所有时间的数据。 -如果只想从一天前的数据开始订阅,而不需要更早的历史数据,可以再加上一个时间条件: +注意,这里没有指定起始时间,所以会读到所有时间的数据。如果只想从一天前的数据开始订阅,而不需要更早的历史数据,可以再加上一个时间条件: ```sql select * from meters where ts > now - 1d and current > 10; ``` -订阅的`topic`实际上是它的名字,因为订阅功能是在客户端API中实现的, -所以没必要保证它全局唯一,但需要它在一台客户端机器上唯一。 +订阅的`topic`实际上是它的名字,因为订阅功能是在客户端API中实现的,所以没必要保证它全局唯一,但需要它在一台客户端机器上唯一。 -如果名`topic`的订阅不存在,参数`restart`没有意义; -但如果用户程序创建这个订阅后退出,当它再次启动并重新使用这个`topic`时, -`restart`就会被用于决定是从头开始读取数据,还是接续上次的位置进行读取。 -本例中,如果`restart`是 **true**(非零值),用户程序肯定会读到所有数据。 -但如果这个订阅之前就存在了,并且已经读取了一部分数据, -且`restart`是 **false**(**0**),用户程序就不会读到之前已经读取的数据了。 +如果名`topic`的订阅不存在,参数`restart`没有意义;但如果用户程序创建这个订阅后退出,当它再次启动并重新使用这个`topic`时,`restart`就会被用于决定是从头开始读取数据,还是接续上次的位置进行读取。本例中,如果`restart`是 **true**(非零值),用户程序肯定会读到所有数据。但如果这个订阅之前就存在了,并且已经读取了一部分数据,且`restart`是 **false**(**0**),用户程序就不会读到之前已经读取的数据了。 -`taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。 -在同步模式下,如过前后两次调用`taos_consume`的时间间隔小于此时间, -`taos_consume`会阻塞,直到间隔超过此时间。 -异步模式下,这个时间是两次调用回调函数的最小时间间隔。 +`taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。在同步模式下,如果前后两次调用`taos_consume`的时间间隔小于此时间,`taos_consume`会阻塞,直到间隔超过此时间。异步模式下,这个时间是两次调用回调函数的最小时间间隔。 -`taos_subscribe`的倒数第二个参数用于用户程序向回调函数传递附加参数, -订阅API不对其做任何处理,只原样传递给回调函数。此参数在同步模式下无意义。 +`taos_subscribe`的倒数第二个参数用于用户程序向回调函数传递附加参数,订阅API不对其做任何处理,只原样传递给回调函数。此参数在同步模式下无意义。 订阅创建以后,就可以消费其数据了,同步模式下,示例代码是下面的 else 部分: @@ -221,9 +161,7 @@ if (async) { } ``` -这里是一个 **while** 循环,用户每按一次回车键就调用一次`taos_consume`, -而`taos_consume`的返回值是查询到的结果集,与`taos_use_result`完全相同, -例子中使用这个结果集的代码是函数`print_result`: +这里是一个 **while** 循环,用户每按一次回车键就调用一次`taos_consume`,而`taos_consume`的返回值是查询到的结果集,与`taos_use_result`完全相同,例子中使用这个结果集的代码是函数`print_result`: ```c void print_result(TAOS_RES* res, int blockFetch) { @@ -249,8 +187,7 @@ void print_result(TAOS_RES* res, int blockFetch) { } ``` -其中的 `taos_print_row` 用于处理订阅到数据,在我们的例子中,它会打印出所有符合条件的记录。 -而异步模式下,消费订阅到的数据则显得更为简单: +其中的 `taos_print_row` 用于处理订阅到数据,在我们的例子中,它会打印出所有符合条件的记录。而异步模式下,消费订阅到的数据则显得更为简单: ```c void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { @@ -264,11 +201,7 @@ void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { taos_unsubscribe(tsub, keep); ``` -其第二个参数,用于决定是否在客户端保留订阅的进度信息。 -如果这个参数是**false**(**0**),那无论下次调用`taos_subscribe`的时的`restart`参数是什么, -订阅都只能重新开始。 -另外,进度信息的保存位置是 *{DataDir}/subscribe/* 这个目录下, -每个订阅有一个与其`topic`同名的文件,删掉某个文件,同样会导致下次创建其对应的订阅时只能重新开始。 +其第二个参数,用于决定是否在客户端保留订阅的进度信息。如果这个参数是**false**(**0**),那无论下次调用`taos_subscribe`时的`restart`参数是什么,订阅都只能重新开始。另外,进度信息的保存位置是 *{DataDir}/subscribe/* 这个目录下,每个订阅有一个与其`topic`同名的文件,删掉某个文件,同样会导致下次创建其对应的订阅时只能重新开始。 代码介绍完毕,我们来看一下实际的运行效果。假设: @@ -291,12 +224,11 @@ $ taos > insert into D1001 values(now, 12, 220, 1); ``` -这时,因为电流超过了10A,您应该可以看到示例程序将它输出到了屏幕上。 -您可以继续插入一些数据观察示例程序的输出。 +这时,因为电流超过了10A,您应该可以看到示例程序将它输出到了屏幕上。您可以继续插入一些数据观察示例程序的输出。 ### Java 使用数据订阅功能 -订阅功能也提供了 Java 开发接口,相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation20/connector/)。需要注意的是,目前 Java 接口没有提供异步订阅模式,但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 +订阅功能也提供了 Java 开发接口,相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation/connector/java#subscribe)。需要注意的是,目前 Java 接口没有提供异步订阅模式,但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 下面以一个示例程序介绍其具体使用方法。它所完成的功能与前面介绍的 C 语言示例基本相同,也是订阅数据库中所有电流超过 10A 的记录。 @@ -406,7 +338,7 @@ ts: 1597466400000 current: 12.4 voltage: 220 phase: 1 location: Beijing.Chaoyang ``` -## 缓存(Cache) +## 缓存(Cache) TDengine采用时间驱动缓存管理策略(First-In-First-Out,FIFO),又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式(Least-Recent-Use,LRU),直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候,将最早的数据批量写入磁盘。一般意义上来说,对于物联网数据的使用,用户最为关心最近产生的数据,即当前状态。TDengine充分利用了这一特性,将最近到达的(当前状态)数据保存在缓存中。 @@ -414,7 +346,7 @@ TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接 TDengine分配固定大小的内存空间作为缓存空间,缓存空间可根据应用的需求和硬件资源配置。通过适当的设置缓存空间,TDengine可以提供极高性能的写入和查询的支持。TDengine中每个虚拟节点(virtual node)创建时分配独立的缓存池。每个虚拟节点管理自己的缓存池,不同虚拟节点间不共享缓存池。每个虚拟节点内部所属的全部表共享该虚拟节点的缓存池。 -TDengine将内存池按块划分进行管理,数据在内存块里按照列式存储。一个vnode的内存池是在vnode创建时按块分配好的,而且每个内存块按照先进先出的原则进行管理。一张表所需要的内存块是从vnode的内存池中进行分配的,块的大小由系统配置参数cache决定。每张表最大内存块的数目由配置参数tblocks决定,每张表平均的内存块的个数由配置参数ablocks决定。因此对于一个vnode, 总的内存大小为: `cache * ablocks * tables`。内存块参数cache不宜过小,一个cache block需要能存储至少几十条以上记录,才会有效率。参数ablocks最小为2,保证每张表平均至少能分配两个内存块。 +TDengine将内存池按块划分进行管理,数据在内存块里是以行(row)的形式存储。一个vnode的内存池是在vnode创建时按块分配好,而且每个内存块按照先进先出的原则进行管理。在创建内存池时,块的大小由系统配置参数cache决定;每个vnode中内存块的数目则由配置参数blocks决定。因此对于一个vnode,总的内存大小为:`cache * blocks`。一个cache block需要保证每张表能存储至少几十条以上记录,才会有效率。 你可以通过函数last_row快速获取一张表或一张超级表的最后一条记录,这样很便于在大屏显示各设备的实时状态或采集值。例如: @@ -425,7 +357,7 @@ select last_row(voltage) from meters where location='Beijing.Chaoyang'; 该SQL语句将获取所有位于北京朝阳区的电表最后记录的电压值。 -## 报警监测(Alert) +## 报警监测(Alert) 在 TDengine 的应用场景中,报警监测是一个常见需求,从概念上说,它要求程序从最近一段时间的数据中筛选出符合一定条件的数据,并基于这些数据根据定义好的公式计算出一个结果,当这个结果符合某个条件且持续一定时间后,以某种形式通知用户。 diff --git a/documentation20/webdocs/markdowndocs/connector-java-ch.md b/documentation20/cn/08.connector/01.java/docs.md similarity index 50% rename from documentation20/webdocs/markdowndocs/connector-java-ch.md rename to documentation20/cn/08.connector/01.java/docs.md index 7ba573d2e44bb3848d13a2f6b7cb81038843a291..3442a2248cd3743cc93034fb5aa9d13b96079543 100644 --- a/documentation20/webdocs/markdowndocs/connector-java-ch.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -1,62 +1,62 @@ # Java Connector -Java连接器支持的系统有: -| **CPU类型** | x64(64bit) | | | ARM64 | ARM32 | -| ------------ | ------------ | -------- | -------- | -------- | -------- | -| **OS类型** | Linux | Win64 | Win32 | Linux | Linux | -| **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** | +TDengine 提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现,可在 maven 的中央仓库 [Sonatype Repository][1] 搜索下载。 -Java连接器的使用请参见视频教程。 +`taos-jdbcdriver` 的实现包括 2 种形式: JDBC-JNI 和 JDBC-RESTful(taos-jdbcdriver-2.0.18 开始支持 JDBC-RESTful)。 JDBC-JNI 通过调用客户端 libtaos.so(或 taos.dll )的本地方法实现, JDBC-RESTful 则在内部封装了 RESTful 接口实现。 -TDengine 为了方便 Java 应用使用,提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。 +![tdengine-connector](page://images/tdengine-jdbc-connector.png) -由于 TDengine 是使用 c 语言开发的,使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 +上图显示了 3 种 Java 应用使用连接器访问 TDengine 的方式: -* libtaos.so - 在 linux 系统中成功安装 TDengine 后,依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so,该目录包含在 Linux 自动扫描路径上,无需单独指定。 - -* taos.dll - 在 windows 系统中安装完客户端之后,驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下,同样无需要单独指定。 +* JDBC-JNI:Java 应用在物理节点1(pnode1)上使用 JDBC-JNI 的 API ,直接调用客户端 API(libtaos.so 或 taos.dll)将写入和查询请求发送到位于物理节点2(pnode2)上的 taosd 实例。 +* RESTful:应用将 SQL 发送给位于物理节点2(pnode2)上的 RESTful 连接器,再调用客户端 API(libtaos.so)。 +* JDBC-RESTful:Java 应用通过 JDBC-RESTful 的 API ,将 SQL 封装成一个 RESTful 请求,发送给物理节点2的 RESTful 连接器。 -> 注意:在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14],Linux 服务器安装完 TDengine 之后默认已安装 client,也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。 +TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致,但时序空间数据库与关系对象型数据库服务的对象和技术特征存在差异,导致 `taos-jdbcdriver` 与传统的 JDBC driver 也存在一定差异。在使用时需要注意以下几点: -TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一致,但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致 taos-jdbcdriver 并未完全实现 JDBC 标准规范。在使用时需要注意以下几点: - -* TDengine 不提供针对单条数据记录的删除和修改的操作,驱动中也没有支持相关方法。 -* 由于不支持删除和修改,所以也不支持事务操作。 +* TDengine 目前不支持针对单条数据记录的删除操作。 +* 目前不支持事务操作。 * 目前不支持表间的 union 操作。 -* 目前不支持嵌套查询(nested query),对每个 Connection 的实例,至多只能有一个打开的 ResultSet 实例;如果在 ResultSet还没关闭的情况下执行了新的查询,TSDBJDBCDriver 则会自动关闭上一个 ResultSet。 - -## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 - -| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | -| --- | --- | --- | -| 2.0.12 及以上 | 2.0.8.0 及以上 | 1.8.x | -| 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | -| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | - -## TDengine DataType 和 Java DataType - -TDengine 目前支持时间戳、数字、字符、布尔类型,与 Java 对应类型转换如下: - -| TDengine DataType | Java DataType | -| --- | --- | -| TIMESTAMP | java.sql.Timestamp | -| INT | java.lang.Integer | -| BIGINT | java.lang.Long | -| FLOAT | java.lang.Float | -| DOUBLE | java.lang.Double | -| SMALLINT, TINYINT |java.lang.Short | -| BOOL | java.lang.Boolean | -| BINARY, NCHAR | java.lang.String | - -## 如何获取 TAOS-JDBCDriver +* 目前不支持嵌套查询(nested query)。 +* 对每个 Connection 的实例,至多只能有一个打开的 ResultSet 实例;如果在 ResultSet 还没关闭的情况下执行了新的查询,taos-jdbcdriver 会自动关闭上一个 ResultSet。 + + +## JDBC-JNI和JDBC-RESTful的对比 + +
    + + + + + + + + + + + + + + + + + + + + + + + + +
    对比项JDBC-JNIJDBC-RESTful
    支持的操作系统linux、windows全平台
    是否需要安装 client需要不需要
    server 升级后是否需要升级 client需要不需要
    写入性能JDBC-RESTful 是 JDBC-JNI 的 50%~90%
    查询性能JDBC-RESTful 与 JDBC-JNI 没有差别
    + + +## 如何获取 taos-jdbcdriver ### maven 仓库 目前 taos-jdbcdriver 已经发布到 [Sonatype Repository][1] 仓库,且各大仓库都已同步。 + * [sonatype][8] * [mvnrepository][9] * [maven.aliyun][10] @@ -67,30 +67,63 @@ maven 项目中使用如下 pom.xml 配置即可: com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.18 ``` ### 源码编译打包 -下载 [TDengine][3] 源码之后,进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。 +下载 [TDengine][3] 源码之后,进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package -Dmaven.test.skip=true` 即可生成相应 jar 包。 -## 使用说明 + + +## JDBC的使用说明 ### 获取连接 -#### 通过JdbcUrl获取连接 +#### 指定URL获取连接 + +通过指定URL获取连接,如下所示: + +```java +Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); +String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` + +以上示例,使用 **JDBC-RESTful** 的 driver,建立了到 hostname 为 taosdemo.com,端口为 6041,数据库名为 test 的连接。这个 URL 中指定用户名(user)为 root,密码(password)为 taosdata。 + +使用 JDBC-RESTful 接口,不需要依赖本地函数库。与 JDBC-JNI 相比,仅需要: + +1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”; +2. jdbcUrl 以“jdbc:TAOS-RS://”开头; +3. 使用 6041 作为连接端口。 + +如果希望获得更好的写入和查询性能,Java 应用可以使用 **JDBC-JNI** 的driver,如下所示: -通过指定的jdbcUrl获取连接,如下所示: ```java Class.forName("com.taosdata.jdbc.TSDBDriver"); String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(jdbcUrl); ``` -以上示例,建立了到hostname为taosdemo.com,端口为6030(TDengine的默认端口),数据库名为test的连接。这个url中指定用户名(user)为root,密码(password)为taosdata。 + +以上示例,使用了 JDBC-JNI 的 driver,建立了到 hostname 为 taosdemo.com,端口为 6030(TDengine 的默认端口),数据库名为 test 的连接。这个 URL 中指定用户名(user)为 root,密码(password)为 taosdata。 + +**注意**:使用 JDBC-JNI 的 driver,taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 + +* libtaos.so + 在 linux 系统中成功安装 TDengine 后,依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so,该目录包含在 Linux 自动扫描路径上,无需单独指定。 + +* taos.dll + 在 windows 系统中安装完客户端之后,驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下,同样无需要单独指定。 + +> 在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14],Linux 服务器安装完 TDengine 之后默认已安装 client,也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。 + +JDBC-JNI 的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 TDengine 的 JDBC URL 规范格式为: -`jdbc:TAOS://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]` +`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]` + url中的配置参数如下: * user:登录 TDengine 用户名,默认值 root。 * password:用户登录密码,默认值 taosdata。 @@ -99,13 +132,17 @@ url中的配置参数如下: * locale:客户端语言环境,默认值系统当前 locale。 * timezone:客户端使用的时区,默认值为系统当前时区。 -#### 使用JdbcUrl和Properties获取连接 -除了通过指定的jdbcUrl获取连接,还可以使用Properties指定建立连接时的参数,如下所示: + +#### 指定URL和Properties获取连接 + +除了通过指定的 URL 获取连接,还可以使用 Properties 指定建立连接时的参数,如下所示: ```java public Connection getConn() throws Exception{ Class.forName("com.taosdata.jdbc.TSDBDriver"); + // Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; + // String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); @@ -114,9 +151,10 @@ public Connection getConn() throws Exception{ return conn; } ``` -以上示例,建立一个到hostname为taosdemo.com,端口为6030,数据库名为test的连接。这个连接在url中指定了用户名(user)为root,密码(password)为taosdata,并在connProps中指定了使用的字符集、语言环境、时区等信息。 -properties中的配置参数如下: +以上示例,建立一个到 hostname 为 taosdemo.com,端口为 6030,数据库名为 test 的连接。注释为使用 JDBC-RESTful 时的方法。这个连接在 url 中指定了用户名(user)为 root,密码(password)为 taosdata,并在 connProps 中指定了使用的字符集、语言环境、时区等信息。 + +properties 中的配置参数如下: * TSDBDriver.PROPERTY_KEY_USER:登录 TDengine 用户名,默认值 root。 * TSDBDriver.PROPERTY_KEY_PASSWORD:用户登录密码,默认值 taosdata。 * TSDBDriver.PROPERTY_KEY_CONFIG_DIR:客户端配置文件目录路径,Linux OS 上默认值 /etc/taos ,Windows OS 上默认值 C:/TDengine/cfg。 @@ -124,10 +162,15 @@ properties中的配置参数如下: * TSDBDriver.PROPERTY_KEY_LOCALE:客户端语言环境,默认值系统当前 locale。 * TSDBDriver.PROPERTY_KEY_TIME_ZONE:客户端使用的时区,默认值为系统当前时区。 + + #### 使用客户端配置文件建立连接 -当使用JDBC连接TDengine集群时,可以使用客户端配置文件,在客户端配置文件中指定集群的firstEp、secondEp参数。 + +当使用 JDBC-JNI 连接 TDengine 集群时,可以使用客户端配置文件,在客户端配置文件中指定集群的 firstEp、secondEp参数。 如下所示: -1. 在java中不指定hostname和port + +1. 在 Java 应用中不指定 hostname 和 port + ```java public Connection getConn() throws Exception{ Class.forName("com.taosdata.jdbc.TSDBDriver"); @@ -140,7 +183,9 @@ public Connection getConn() throws Exception{ return conn; } ``` -2. 在配置文件中指定firstEp和secondEp + +2. 在配置文件中指定 firstEp 和 secondEp + ``` # first fully qualified domain name (FQDN) for TDengine system firstEp cluster_node1:6030 @@ -149,23 +194,25 @@ firstEp cluster_node1:6030 secondEp cluster_node2:6030 # default system charset -# charset UTF-8 +# charset UTF-8 # system locale # locale en_US.UTF-8 ``` -以上示例,jdbc会使用客户端的配置文件,建立到hostname为cluster_node1,端口为6030,数据库名为test的连接。当集群中firstEp节点失效时,JDBC会尝试使用secondEp连接集群。 -TDengine中,只要保证firstEp和secondEp中一个节点有效,就可以正常建立到集群的连接。 +以上示例,jdbc 会使用客户端的配置文件,建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时,JDBC 会尝试使用 secondEp 连接集群。 +TDengine 中,只要保证 firstEp 和 secondEp 中一个节点有效,就可以正常建立到集群的连接。 -> 注意:这里的配置文件指的是调用JDBC Connector的应用程序所在机器上的配置文件,Linux OS 上默认值 /etc/taos/taos.cfg ,Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 +> 注意:这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件,Linux OS 上默认值 /etc/taos/taos.cfg ,Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 #### 配置参数的优先级 -通过以上3种方式获取连接,如果配置参数在url、Properties、客户端配置文件中有重复,则参数的`优先级由高到低`分别如下: + +通过以上 3 种方式获取连接,如果配置参数在 url、Properties、客户端配置文件中有重复,则参数的`优先级由高到低`分别如下: 1. JDBC URL 参数,如上所述,可以在 JDBC URL 的参数中指定。 2. Properties connProps 3. 客户端配置文件 taos.cfg -例如:在url中指定了password为taosdata,在Properties中指定了password为taosdemo,那么,JDBC会使用url中的password建立连接。 + +例如:在 url 中指定了 password 为 taosdata,在 Properties 中指定了 password 为 taosdemo,那么,JDBC 会使用 url 中的 password 建立连接。 > 更多详细配置请参考[客户端配置][13] @@ -183,6 +230,7 @@ stmt.executeUpdate("use db"); // create table stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)"); ``` + > 注意:如果不使用 `use db` 指定数据库,则后续对表的操作都需要增加数据库名称作为前缀,如 db.tb。 ### 插入数据 @@ -193,6 +241,7 @@ int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now System.out.println("insert " + affectedRows + " rows."); ``` + > now 为系统内部函数,默认为服务器当前时间。 > `now + 1s` 代表服务器当前时间往后加 1 秒,数字后面代表时间单位:a(毫秒), s(秒), m(分), h(小时), d(天),w(周), n(月), y(年)。 @@ -214,9 +263,29 @@ while(resultSet.next()){ System.out.printf("%s, %d, %s\n", ts, temperature, humidity); } ``` + > 查询和操作关系型数据库一致,使用下标获取返回字段内容时从 1 开始,建议使用字段名称获取。 -### 订阅 +### 处理异常 +在报错后,通过SQLException可以获取到错误的信息和错误码: +```java +try (Statement statement = connection.createStatement()) { + // executeQuery + ResultSet resultSet = statement.executeQuery(sql); + // print result + printResult(resultSet); +} catch (SQLException e) { + System.out.println("ERROR Message: " + e.getMessage()); + System.out.println("ERROR Code: " + e.getErrorCode()); + e.printStackTrace(); +} +``` +JDBC连接器可能报错的错误码包括3种:JDBC driver本身的报错(错误码在0x2301到0x2350之间),JNI方法的报错(错误码在0x2351到0x2400之间),TDengine其他功能模块的报错。 +具体的错误码请参考: +* https://github.com/taosdata/TDengine/blob/develop/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java +* https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h + +### 订阅 #### 创建 @@ -248,7 +317,7 @@ while(true) { } ``` -`consume` 方法返回一个结果集,其中包含从上次 `consume` 到目前为止的所有新数据。请务必按需选择合理的调用 `consume` 的频率(如例子中的`Thread.sleep(1000)`),否则会给服务端造成不必要的压力。 +`consume` 方法返回一个结果集,其中包含从上次 `consume` 到目前为止的所有新数据。请务必按需选择合理的调用 `consume` 的频率(如例子中的 `Thread.sleep(1000)`),否则会给服务端造成不必要的压力。 #### 关闭订阅 @@ -265,13 +334,17 @@ resultSet.close(); stmt.close(); conn.close(); ``` + > `注意务必要将 connection 进行关闭`,否则会出现连接泄露。 + + ## 与连接池使用 **HikariCP** * 引入相应 HikariCP maven 依赖: + ```xml com.zaxxer @@ -281,19 +354,21 @@ conn.close(); ``` * 使用示例如下: + ```java public static void main(String[] args) throws SQLException { HikariConfig config = new HikariConfig(); + // jdbc properties config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/log"); config.setUsername("root"); config.setPassword("taosdata"); - - config.setMinimumIdle(3); //minimum number of idle connection + // connection pool configurations + config.setMinimumIdle(10); //minimum number of idle connection config.setMaximumPoolSize(10); //maximum number of connection in the pool - config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool - config.setIdleTimeout(60000); // max idle time for recycle idle connection - config.setConnectionTestQuery("describe log.dn"); //validation query - config.setValidationTimeout(3000); //validation query timeout + config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setMaxLifetime(0); // maximum life time for each connection + config.setIdleTimeout(0); // max idle time for recycle idle connection + config.setConnectionTestQuery("select server_status()"); //validation query HikariDataSource ds = new HikariDataSource(config); //create datasource @@ -306,6 +381,7 @@ conn.close(); connection.close(); // put back to conneciton pool } ``` + > 通过 HikariDataSource.getConnection() 获取连接后,使用完成后需要调用 close() 方法,实际上它并不会关闭连接,只是放回连接池中。 > 更多 HikariCP 使用问题请查看[官方说明][5] @@ -322,40 +398,32 @@ conn.close(); ``` * 使用示例如下: + ```java public static void main(String[] args) throws Exception { - Properties properties = new Properties(); - properties.put("driverClassName","com.taosdata.jdbc.TSDBDriver"); - properties.put("url","jdbc:TAOS://127.0.0.1:6030/log"); - properties.put("username","root"); - properties.put("password","taosdata"); - - properties.put("maxActive","10"); //maximum number of connection in the pool - properties.put("initialSize","3");//initial number of connection - properties.put("maxWait","10000");//maximum wait milliseconds for get connection from pool - properties.put("minIdle","3");//minimum number of connection in the pool - - properties.put("timeBetweenEvictionRunsMillis","3000");// the interval milliseconds to test connection - - properties.put("minEvictableIdleTimeMillis","60000");//the minimum milliseconds to keep idle - properties.put("maxEvictableIdleTimeMillis","90000");//the maximum milliseconds to keep idle - properties.put("validationQuery","describe log.dn"); //validation query - properties.put("testWhileIdle","true"); // test connection while idle - properties.put("testOnBorrow","false"); // don't need while testWhileIdle is true - properties.put("testOnReturn","false"); // don't need while testWhileIdle is true - - //create druid datasource - DataSource ds = DruidDataSourceFactory.createDataSource(properties); - Connection connection = ds.getConnection(); // get connection + DruidDataSource dataSource = new DruidDataSource(); + // jdbc properties + dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + dataSource.setUrl(url); + dataSource.setUsername("root"); + dataSource.setPassword("taosdata"); + // pool configurations + dataSource.setInitialSize(10); + dataSource.setMinIdle(10); + dataSource.setMaxActive(10); + dataSource.setMaxWait(30000); + dataSource.setValidationQuery("select server_status()"); + + Connection connection = dataSource.getConnection(); // get connection Statement statement = connection.createStatement(); // get statement - //query or insert // ... connection.close(); // put back to conneciton pool } ``` + > 更多 druid 使用问题请查看[官方说明][6] **注意事项** @@ -370,18 +438,54 @@ server_status()| Query OK, 1 row(s) in set (0.000141s) ``` + + ## 与框架使用 * Spring JdbcTemplate 中使用 taos-jdbcdriver,可参考 [SpringJdbcTemplate][11] * Springboot + Mybatis 中使用,可参考 [springbootdemo][12] + + +## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 + +| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | +| -------------------- | ----------------- | -------- | +| 2.0.22 | 2.0.18.0 及以上 | 1.8.x | +| 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.0 | 1.8.x | +| 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | +| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | + + + +## TDengine DataType 和 Java DataType + +TDengine 目前支持时间戳、数字、字符、布尔类型,与 Java 对应类型转换如下: + +| TDengine DataType | Java DataType | +| ----------------- | ------------------ | +| TIMESTAMP | java.sql.Timestamp | +| INT | java.lang.Integer | +| BIGINT | java.lang.Long | +| FLOAT | java.lang.Float | +| DOUBLE | java.lang.Double | +| SMALLINT | java.lang.Short | +| TINYINT | java.lang.Byte | +| BOOL | java.lang.Boolean | +| BINARY | byte array | +| NCHAR | java.lang.String | + + + ## 常见问题 * java.lang.UnsatisfiedLinkError: no taos in java.library.path **原因**:程序没有找到依赖的本地函数库 taos。 - **解决方法**:windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下,linux 下将建立如下软链 ` ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 + **解决方法**:windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下,linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 * java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform @@ -403,6 +507,7 @@ Query OK, 1 row(s) in set (0.000141s) [10]: https://maven.aliyun.com/mvn/search [11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate [12]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo -[13]: https://www.taosdata.com/cn/documentation20/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE +[13]: https://www.taosdata.com/cn/documentation/administrator/#client [14]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client -[15]: https://www.taosdata.com/cn/getting-started/#%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B +[15]: https://www.taosdata.com/cn/getting-started/#%E5%AE%A2%E6%88%B7%E7%AB%AF + diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/cn/08.connector/docs.md similarity index 85% rename from documentation20/webdocs/markdowndocs/connector-ch.md rename to documentation20/cn/08.connector/docs.md index da9c2e5a119e56243e517e9de0e248184d3c3c93..fb50e20e513aaf9b3791989c76c36eaca8df8f2b 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/cn/08.connector/docs.md @@ -2,7 +2,7 @@ TDengine提供了丰富的应用程序开发接口,其中包括C/C++、Java、Python、Go、Node.js、C# 、RESTful 等,便于用户快速开发应用。 -![image-connecotr](../assets/connector.png) +![image-connecotr](page://images/connector.png) 目前TDengine的连接器可支持的平台广泛,包括:X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台,以及Linux/Win64/Win32等开发环境。对照矩阵如下: @@ -24,8 +24,9 @@ TDengine提供了丰富的应用程序开发接口,其中包括C/C++、Java、 * 在没有安装TDengine服务端软件的系统中使用连接器(除RESTful外)访问 TDengine 数据库,需要安装相应版本的客户端安装包来使应用驱动(Linux系统中文件名为libtaos.so,Windows系统中为taos.dll)被安装在系统中,否则会产生无法找到相应库文件的错误。 * 所有执行 SQL 语句的 API,例如 C/C++ Connector 中的 `tao_query`、`taos_query_a`、`taos_subscribe` 等,以及其它语言中与它们对应的API,每次都只能执行一条 SQL 语句,如果实际参数中包含了多条语句,它们的行为是未定义的。 * 升级到TDengine到2.0.8.0版本的用户,必须更新JDBC连接TDengine必须升级taos-jdbcdriver到2.0.12及以上。 +* 无论选用何种编程语言的连接器,2.0 及以上版本的 TDengine 推荐数据库应用的每个线程都建立一个独立的连接,或基于线程建立连接池,以避免连接内的“USE statement”状态量在线程之间相互干扰(但连接的查询和写入操作都是线程安全的)。 -## 安装连接器驱动步骤 +## 安装连接器驱动步骤 服务器应该已经安装TDengine服务端安装包。连接器驱动安装步骤如下: @@ -136,7 +137,7 @@ taos> taos> ``` -## C/C++ Connector +## C/C++ Connector **C/C++连接器支持的系统有**: @@ -156,7 +157,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时,需要包含TDengine * 在编译时需要链接TDengine动态库。Linux 为 *libtaos.so* ,安装后,位于 _/usr/local/taos/driver_。Windows为 taos.dll,安装后位于 *C:\TDengine*。 * 如未特别说明,当API的返回值是整数时,_0_ 代表成功,其它是代表失败原因的错误码,当返回值是指针时, _NULL_ 表示失败。 -使用C/C++连接器的示例代码请参见 https://github.com/taosdata/TDengine/tree/develop/tests/examples/c。 +使用C/C++连接器的示例代码请参见 https://github.com/taosdata/TDengine/tree/develop/tests/examples/c 。 ### 基础API @@ -208,7 +209,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时,需要包含TDengine - `TAOS_RES* taos_query(TAOS *taos, const char *sql)` - 该API用来执行SQL语句,可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。返回值 NULL 表示失败。 + 该API用来执行SQL语句,可以是DQL、DML或DDL语句。 其中的`taos`参数是通过`taos_connect`获得的指针。不能通过返回值是否是 NULL 来判断执行结果是否失败,而是需要用`taos_errno`函数解析结果集中的错误代码来进行判断。 - `int taos_result_precision(TAOS_RES *res)` @@ -238,13 +239,13 @@ C/C++的API类似于MySQL的C API。应用程序使用时,需要包含TDengine 获取查询结果集每列数据的属性(数据类型、名字、字节数),与taos_num_fileds配合使用,可用来解析`taos_fetch_row`返回的一个元组(一行)的数据。 `TAOS_FIELD` 的结构如下: - ```c - typedef struct taosField { - char name[65]; // 列名 - uint8_t type; // 数据类型 - int16_t bytes; // 字节数 - } TAOS_FIELD; - ``` +```c +typedef struct taosField { + char name[65]; // 列名 + uint8_t type; // 数据类型 + int16_t bytes; // 字节数 +} TAOS_FIELD; +``` - `void taos_stop_query(TAOS_RES *res)` @@ -266,7 +267,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时,需要包含TDengine ### 异步查询API -同步API之外,TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下,异步API处理数据插入的速度比同步API快2\~4倍。异步API采用非阻塞式的调用方式,在系统真正完成某个具体数据库操作前,立即返回。调用的线程可以去处理其他工作,从而可以提升整个应用的性能。异步API在网络延迟严重的情况下,优点尤为突出。 +同步API之外,TDengine还提供性能更高的异步调用API处理数据插入、查询操作。在软硬件环境相同的情况下,异步API处理数据插入的速度比同步API快2~4倍。异步API采用非阻塞式的调用方式,在系统真正完成某个具体数据库操作前,立即返回。调用的线程可以去处理其他工作,从而可以提升整个应用的性能。异步API在网络延迟严重的情况下,优点尤为突出。 异步API都需要应用提供相应的回调函数,回调函数参数设置如下:前两个参数都是一致的,第三个参数依不同的API而定。第一个参数param是应用调用异步API时提供给系统的,用于回调时,应用能够找回具体操作的上下文,依具体实现而定。第二个参数是SQL操作的结果集,如果为空,比如insert操作,表示没有记录返回,如果不为空,比如select操作,表示有记录返回。 @@ -288,13 +289,6 @@ C/C++的API类似于MySQL的C API。应用程序使用时,需要包含TDengine * res:`taos_query_a`回调时返回的结果集 * fp:回调函数。其参数`param`是用户可定义的传递给回调函数的参数结构体;`numOfRows`是获取到的数据的行数(不是整个查询结果集的函数)。 在回调函数中,应用可以通过调用`taos_fetch_row`前向迭代获取批量记录中每一行记录。读完一块内的所有记录后,应用需要在回调函数中继续调用`taos_fetch_rows_a`获取下一批记录进行处理,直到返回的记录数(numOfRows)为零(结果返回完成)或记录数为负值(查询出错)。 -- `void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);` - - 异步获取一条记录。其中: - - * res:`taos_query_a`回调时返回的结果集 - * fp:回调函数。其参数`param`是应用提供的一个用于回调的参数。回调时,第三个参数`row`指向一行记录。不同于`taos_fetch_rows_a`,应用无需调用`taos_fetch_row`来获取一行数据,更加简单,但数据提取性能不及批量获取的API。 - TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线程同时打开多张表,并可以同时对每张打开的表进行查询或者插入操作。需要指出的是,**客户端应用必须确保对同一张表的操作完全串行化**,即对同一个表的插入或查询操作未完成时(未返回时),不能够执行第二个插入或查询操作。 ### 参数绑定API @@ -313,17 +307,17 @@ TDengine的异步API均采用非阻塞调用模式。应用程序可以用多线 进行参数绑定,bind指向一个数组,需保证此数组的元素数量和顺序与sql语句中的参数完全一致。TAOS_BIND 的使用方法与 MySQL中的 MYSQL_BIND 一致,具体定义如下: - ```c - typedef struct TAOS_BIND { - int buffer_type; - void * buffer; - unsigned long buffer_length; // 未实际使用 - unsigned long *length; - int * is_null; - int is_unsigned; // 未实际使用 - int * error; // 未实际使用 - } TAOS_BIND; - ``` +```c +typedef struct TAOS_BIND { + int buffer_type; + void * buffer; + unsigned long buffer_length; // 未实际使用 + unsigned long *length; + int * is_null; + int is_unsigned; // 未实际使用 + int * error; // 未实际使用 +} TAOS_BIND; +``` - `int taos_stmt_add_batch(TAOS_STMT *stmt)` @@ -392,12 +386,12 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 取消订阅。 如参数 `keepProgress` 不为0,API会保留订阅的进度信息,后续调用 `taos_subscribe` 时可以基于此进度继续;否则将删除进度信息,后续只能重新开始读取数据。 -## Python Connector +## Python Connector -Python连接器的使用参见视频教程 +Python连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1963.html) ### 安装准备 -* 应用驱动安装请参考安装连接器驱动步骤。 +* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 * 已安装python 2.7 or >= 3.4 * 已安装pip 或 pip3 @@ -438,6 +432,7 @@ python -m pip install python3\ import taos ``` * 获取连接并获取游标对象 + ```python conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos") c1 = conn.cursor() @@ -445,6 +440,7 @@ c1 = conn.cursor() * host 是TDengine 服务端所有IP, config 为客户端配置文件所在目录 * 写入数据 + ```python import datetime @@ -466,6 +462,7 @@ affected_rows = c1.execute(' '.join(sqlcmd)) ``` * 查询数据 + ```python c1.execute('select * from tb') # 拉取查询结果 @@ -483,6 +480,7 @@ for data in c1: ``` * 创建订阅 + ```python # 创建一个主题为 'test' 消费周期为1000毫秒的订阅 # 第一个参数为 True 表示重新开始订阅,如为 False 且之前创建过主题为 'test' 的订阅,则表示继续消费此订阅的数据,而不是重新开始消费所有数据 @@ -490,6 +488,7 @@ sub = conn.subscribe(True, "test", "select * from tb;", 1000) ``` * 消费订阅的数据 + ```python data = sub.consume() for d in data: @@ -497,15 +496,18 @@ for d in data: ``` * 取消订阅 + ```python sub.close() ``` * 关闭连接 + ```python c1.close() conn.close() ``` + #### 帮助信息 用户可通过python的帮助信息直接查看模块的使用信息,或者参考tests/examples/python中的示例程序。以下为部分常用类和方法: @@ -525,6 +527,7 @@ conn.close() 用于生成taos.TDengineConnection的实例。 ### Python客户端使用示例代码 + 在tests/examples/python中,我们提供了一个示例Python程序read_example.py,可以参考这个程序来设计用户自己的写入、查询程序。在安装了对应的客户端后,通过import taos引入taos类。主要步骤如下 - 通过taos.connect获取TDengineConnection对象,这个对象可以一个程序只申请一个,在多线程中共享。 @@ -534,9 +537,9 @@ conn.close() - 如果执行的是查询语句,则execute执行成功后,需要通过fetchall方法去拉取结果集。 具体方法可以参考示例代码。 -## RESTful Connector +## RESTful Connector -为支持各种不同类型平台的开发,TDengine提供符合REST设计标准的API,即RESTful API。为最大程度降低学习成本,不同于其他数据库RESTful API的设计方法,TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库,仅需要一个URL。RESTful连接器的使用参见视频教程。 +为支持各种不同类型平台的开发,TDengine提供符合REST设计标准的API,即RESTful API。为最大程度降低学习成本,不同于其他数据库RESTful API的设计方法,TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库,仅需要一个URL。RESTful连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1965.html)。 ### HTTP请求格式 @@ -588,7 +591,8 @@ curl -u username:password -d '' :/rest/sql ```json { "status": "succ", - "head": ["Time Stamp","current", …], + "head": ["ts","current", …], + "column_meta": [["ts",9,8],["current",6,4], …], "data": [ ["2018-10-03 14:38:05.000", 10.3, …], ["2018-10-03 14:38:15.000", 12.6, …] @@ -599,10 +603,23 @@ curl -u username:password -d '' :/rest/sql 说明: -- status: 告知操作结果是成功还是失败 -- head: 表的定义,如果不返回结果集,仅有一列“affected_rows” -- data: 具体返回的数据,一排一排的呈现,如果不返回结果集,仅[[affected_rows]] -- rows: 表明总共多少行数据 +- status: 告知操作结果是成功还是失败。 +- head: 表的定义,如果不返回结果集,则仅有一列“affected_rows”。(从 2.0.17 版本开始,建议不要依赖 head 返回值来判断数据列类型,而推荐使用 column_meta。在未来版本中,有可能会从返回值中去掉 head 这一项。) +- column_meta: 从 2.0.17 版本开始,返回值中增加这一项来说明 data 里每一列的数据类型。具体每个列会用三个值来说明,分别为:列名、列类型、类型长度。例如`["current",6,4]`表示列名为“current”;列类型为 6,也即 float 类型;类型长度为 4,也即对应 4 个字节表示的 float。如果列类型为 binary 或 nchar,则类型长度表示该列最多可以保存的内容长度,而不是本次返回值中的具体数据长度。当列类型是 nchar 的时候,其类型长度表示可以保存的 unicode 字符数量,而不是 bytes。 +- data: 具体返回的数据,一行一行的呈现,如果不返回结果集,那么就仅有[[affected_rows]]。data 中每一行的数据列顺序,与 column_meta 中描述数据列的顺序完全一致。 +- rows: 表明总共多少行数据。 + +column_meta 中的列类型说明: +* 1:BOOL +* 2:TINYINT +* 3:SMALLINT +* 4:INT +* 5:BIGINT +* 6:FLOAT +* 7:DOUBLE +* 8:BINARY +* 9:TIMESTAMP +* 10:NCHAR ### 自定义授权码 @@ -648,7 +665,8 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 ```json { "status": "succ", - "head": ["Time Stamp","current","voltage","phase"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ ["2018-10-03 14:38:05.000",10.3,219,0.31], ["2018-10-03 14:38:15.000",12.6,218,0.33] @@ -668,8 +686,9 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 19 { "status": "succ", "head": ["affected_rows"], + "column_meta": [["affected_rows",4,4]], "data": [[1]], - "rows": 1, + "rows": 1 } ``` @@ -688,7 +707,8 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 ```json { "status": "succ", - "head": ["column1","column2","column3"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ [1538548685000,10.3,219,0.31], [1538548695000,12.6,218,0.33] @@ -709,7 +729,8 @@ HTTP请求URL采用`sqlutc`时,返回结果集的时间戳将采用UTC时间 ```json { "status": "succ", - "head": ["column1","column2","column3"], + "head": ["ts","current","voltage","phase"], + "column_meta": [["ts",9,8],["current",6,4],["voltage",4,4],["phase",6,4]], "data": [ ["2018-10-03T14:38:05.000+0800",10.3,219,0.31], ["2018-10-03T14:38:15.000+0800",12.6,218,0.33] @@ -720,21 +741,21 @@ HTTP请求URL采用`sqlutc`时,返回结果集的时间戳将采用UTC时间 ### 重要配置项 -下面仅列出一些与RESTFul接口有关的配置参数,其他系统参数请看配置文件里的说明。注意:配置修改后,需要重启taosd服务才能生效 +下面仅列出一些与RESTful接口有关的配置参数,其他系统参数请看配置文件里的说明。注意:配置修改后,需要重启taosd服务才能生效 -- httpPort: 对外提供RESTFul服务的端口号,默认绑定到6041 -- httpMaxThreads: 启动的线程数量,默认为2 +- httpPort: 对外提供RESTful服务的端口号,默认绑定到6041 +- httpMaxThreads: 启动的线程数量,默认为2(2.0.17版本开始,默认值改为CPU核数的一半向下取整) - restfulRowLimit: 返回结果集(JSON格式)的最大条数,默认值为10240 - httpEnableCompress: 是否支持压缩,默认不支持,目前TDengine仅支持gzip压缩格式 - httpDebugFlag: 日志开关,131:仅错误和报警信息,135:调试信息,143:非常详细的调试信息,默认131 -## CSharp Connector +## CSharp Connector C#连接器支持的系统有:Linux 64/Windows x64/Windows x86 ### 安装准备 -* 应用驱动安装请参考安装连接器驱动步骤。 +* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 * .NET接口文件TDengineDrivercs.cs和参考程序示例TDengineTest.cs均位于Windows客户端install_directory/examples/C#目录下。 * 在Windows系统上,C#应用程序可以使用TDengine的原生C接口来执行所有数据库操作,后续版本将提供ORM(dapper)框架驱动。 @@ -773,15 +794,15 @@ https://github.com/maikebing/Maikebing.EntityFrameworkCore.Taos https://www.taosdata.com/blog/2020/11/02/1901.html ``` -## Go Connector +## Go Connector ### 安装准备 -* 应用驱动安装请参考安装连接器驱动步骤。 +* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 TDengine提供了GO驱动程序`taosSql`。 `taosSql`实现了GO语言的内置接口`database/sql/driver`。用户只需按如下方式引入包就可以在应用程序中访问TDengine, 详见`https://github.com/taosdata/driver-go/blob/develop/taosSql/driver_test.go`。 -使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go 以及视频教程。 +使用 Go 连接器的示例代码请参考 https://github.com/taosdata/TDengine/tree/develop/tests/examples/go 以及[视频教程](https://www.taosdata.com/blog/2020/11/11/1951.html)。 ```Go import ( @@ -828,7 +849,7 @@ go env -w GOPROXY=https://goproxy.io,direct sql.Open内置的方法,Close closes the statement. -## Node.js Connector +## Node.js Connector Node.js连接器支持的系统有: @@ -837,47 +858,47 @@ Node.js连接器支持的系统有: | **OS类型** | Linux | Win64 | Win32 | Linux | Linux | | **支持与否** | **支持** | **支持** | **支持** | **支持** | **支持** | -Node.js连接器的使用参见视频教程 +Node.js连接器的使用参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1957.html) ### 安装准备 -* 应用驱动安装请参考安装连接器驱动步骤。 +* 应用驱动安装请参考[安装连接器驱动步骤](https://www.taosdata.com/cn/documentation/connector#driver)。 ### 安装Node.js连接器 -用户可以通过npm来进行安装,也可以通过源代码*src/connector/nodejs/* 来进行安装。具体安装步骤如下: +用户可以通过[npm](https://www.npmjs.com/)来进行安装,也可以通过源代码*src/connector/nodejs/* 来进行安装。具体安装步骤如下: -首先,通过npm安装node.js 连接器. +首先,通过[npm](https://www.npmjs.com/)安装node.js 连接器. ```bash npm install td2.0-connector ``` 我们建议用户使用npm 安装node.js连接器。如果您没有安装npm, 可以将*src/connector/nodejs/*拷贝到您的nodejs 项目目录下 -我们使用node-gyp和TDengine服务端进行交互。安装node.js 连接器之前,还需安装以下软件: +我们使用[node-gyp](https://github.com/nodejs/node-gyp)和TDengine服务端进行交互。安装node.js 连接器之前,还需安装以下软件: ### Linux - `python` (建议`v2.7` , `v3.x.x` 目前还不支持) -- `node` 必须采用v10.x版本,其他版本存在包兼容性的问题。 +- `node` 2.0.6支持v12.x和v10.x,2.0.5及更早版本支持v10.x版本,其他版本可能存在包兼容性的问题。 - `make` -- c语言编译器比如GCC +- c语言编译器比如[GCC](https://gcc.gnu.org) ### Windows #### 安装方法1 -使用微软的windows-build-tools在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具 +使用微软的[windows-build-tools](https://github.com/felixrieseberg/windows-build-tools)在`cmd` 命令行界面执行`npm install --global --production windows-build-tools` 即可安装所有的必备工具 #### 安装方法2 手动安装以下工具: -- 安装Visual Studio相关:Visual Studio 2017 Community -- 安装 Python 2.7(`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` +- 安装Visual Studio相关:[Visual Studio Build 工具](https://visualstudio.microsoft.com/thank-you-downloading-visual-studio/?sku=BuildTools) 或者 [Visual Studio 2017 Community](https://visualstudio.microsoft.com/pl/thank-you-downloading-visual-studio/?sku=Community) +- 安装 [Python](https://www.python.org/downloads/) 2.7(`v3.x.x` 暂不支持) 并执行 `npm config set python python2.7` - 进入`cmd`命令行界面, `npm config set msvs_version 2017` -如果以上步骤不能成功执行, 可以参考微软的node.js用户手册Microsoft's Node.js Guidelines for Windows +如果以上步骤不能成功执行, 可以参考微软的node.js用户手册[Microsoft's Node.js Guidelines for Windows](https://github.com/Microsoft/nodejs-guidelines/blob/master/windows-environment.md#compiling-native-addon-modules) 如果在Windows 10 ARM 上使用ARM64 Node.js, 还需添加 "Visual C++ compilers and libraries for ARM64" 和 "Visual C++ ATL for ARM64". @@ -894,7 +915,7 @@ Node-example-raw.js 验证方法: -1. 新建安装验证目录,例如:\~/tdengine-test,拷贝github上nodejsChecker.js源程序。下载地址:(https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js)。 +1. 新建安装验证目录,例如:`~/tdengine-test`,拷贝github上nodejsChecker.js源程序。下载地址:(https://github.com/taosdata/TDengine/tree/develop/tests/examples/nodejs/nodejsChecker.js)。 2. 在命令中执行以下命令: @@ -908,8 +929,7 @@ node nodejsChecker.js host=localhost ### Node.js连接器的使用 -(http://docs.taosdata.com/node) -以下是node.js 连接器的一些基本使用方法,详细的使用方法可参考该文档 +以下是node.js 连接器的一些基本使用方法,详细的使用方法可参考[TDengine Node.js connector](http://docs.taosdata.com/node) #### 建立连接 @@ -980,6 +1000,7 @@ promise.then(function(result) { }) ``` #### 异步函数 + 异步查询数据库的操作和上面类似,只需要在`cursor.execute`, `TaosQuery.execute`等函数后面加上`_a`。 ```javascript var promise1 = cursor.query('select count(*), avg(v1), avg(v2) from meter1;').execute_a() @@ -993,6 +1014,7 @@ promise2.then(function(result) { ``` ### 示例 -这里提供了一个使用NodeJS 连接器建表,插入天气数据并查询插入的数据的代码示例 -这里同样是一个使用NodeJS 连接器建表,插入天气数据并查询插入的数据的代码示例,但和上面不同的是,该示例只使用`cursor`. +[node-example.js](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example.js)提供了一个使用NodeJS 连接器建表,插入天气数据并查询插入的数据的代码示例 + +[node-example-raw.js](https://github.com/taosdata/TDengine/tree/master/tests/examples/nodejs/node-example-raw.js)同样是一个使用NodeJS 连接器建表,插入天气数据并查询插入的数据的代码示例,但和上面不同的是,该示例只使用`cursor`. diff --git a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md b/documentation20/cn/09.connections/docs.md similarity index 82% rename from documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md rename to documentation20/cn/09.connections/docs.md index 4e55f3c5e8f12f3277898b1f825dca5e22b2e6d1..79380f3bbd9680120f63f89a0bfbe6f31f5c7a74 100644 --- a/documentation20/webdocs/markdowndocs/Connections with other Tools-ch.md +++ b/documentation20/cn/09.connections/docs.md @@ -1,7 +1,7 @@ # 与其他工具的连接 -## Grafana +## Grafana TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/)快速集成搭建数据监测报警系统,整个过程无需任何代码开发,TDengine中数据表中内容可以在仪表盘(DashBoard)上进行可视化展现。 @@ -13,7 +13,11 @@ TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/) TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin目录下。 -以CentOS 7.2操作系统为例,将tdengine目录拷贝到/var/lib/grafana/plugins目录下,重新启动grafana即可。 +以CentOS 7.2操作系统为例,将grafanaplugin目录拷贝到/var/lib/grafana/plugins目录下,重新启动grafana即可。 + +```bash +sudo cp -rf /usr/local/taos/connector/grafanaplugin /var/lib/grafana/tdengine +``` ### 使用 Grafana @@ -21,15 +25,15 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 用户可以直接通过 localhost:3000 的网址,登录 Grafana 服务器(用户名/密码:admin/admin),通过左侧 `Configuration -> Data Sources` 可以添加数据源,如下图所示: -![img](../assets/add_datasource1.jpg) +![img](page://images/connections/add_datasource1.jpg) 点击 `Add data source` 可进入新增数据源页面,在查询框中输入 TDengine 可选择添加,如下图所示: -![img](../assets/add_datasource2.jpg) +![img](page://images/connections/add_datasource2.jpg) 进入数据源配置页面,按照默认提示修改相应配置即可: -![img](../assets/add_datasource3.jpg) +![img](page://images/connections/add_datasource3.jpg) * Host: TDengine 集群的中任意一台服务器的 IP 地址与 TDengine RESTful 接口的端口号(6041),默认 http://localhost:6041 * User:TDengine 用户名。 @@ -37,13 +41,13 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 点击 `Save & Test` 进行测试,成功会有如下提示: -![img](../assets/add_datasource4.jpg) +![img](page://images/connections/add_datasource4.jpg) #### 创建 Dashboard 回到主界面创建 Dashboard,点击 Add Query 进入面板查询页面: -![img](../assets/create_dashboard1.jpg) +![img](page://images/connections/create_dashboard1.jpg) 如上图所示,在 Query 中选中 `TDengine` 数据源,在下方查询框可输入相应 sql 进行查询,具体说明如下: @@ -54,7 +58,7 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 按照默认提示查询当前 TDengine 部署所在服务器指定间隔系统内存平均使用量如下: -![img](../assets/create_dashboard2.jpg) +![img](page://images/connections/create_dashboard2.jpg) > 关于如何使用Grafana创建相应的监测界面以及更多有关使用Grafana的信息,请参考Grafana官方的[文档](https://grafana.com/docs/)。 @@ -64,14 +68,14 @@ TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin 点击左侧 `Import` 按钮,并上传 `tdengine-grafana.json` 文件: -![img](../assets/import_dashboard1.jpg) +![img](page://images/connections/import_dashboard1.jpg) 导入完成之后可看到如下效果: -![img](../assets/import_dashboard2.jpg) +![img](page://images/connections/import_dashboard2.jpg) -## Matlab +## Matlab MatLab可以通过安装包内提供的JDBC Driver直接连接到TDengine获取数据到本地工作空间。 @@ -82,12 +86,15 @@ MatLab的适配有下面几个步骤,下面以Windows10上适配MatLab2017a为 - 将TDengine安装包内的驱动程序JDBCDriver-1.0.0-dist.jar拷贝到${matlab_root}\MATLAB\R2017a\java\jar\toolbox - 将TDengine安装包内的taos.lib文件拷贝至${matlab_ root _dir}\MATLAB\R2017a\lib\win64 - 将新添加的驱动jar包加入MatLab的classpath。在${matlab_ root _dir}\MATLAB\R2017a\toolbox\local\classpath.txt文件中添加下面一行 - -​ `$matlabroot/java/jar/toolbox/JDBCDriver-1.0.0-dist.jar` - +​ +``` +$matlabroot/java/jar/toolbox/JDBCDriver-1.0.0-dist.jar +``` - 在${user_home}\AppData\Roaming\MathWorks\MATLAB\R2017a\下添加一个文件javalibrarypath.txt, 并在该文件中添加taos.dll的路径,比如您的taos.dll是在安装时拷贝到了C:\Windows\System32下,那么就应该在javalibrarypath.txt中添加如下一行: - -​ `C:\Windows\System32` +​ +``` +C:\Windows\System32 +``` ### 在MatLab中连接TDengine获取数据 @@ -95,23 +102,25 @@ MatLab的适配有下面几个步骤,下面以Windows10上适配MatLab2017a为 - 创建一个连接: - `conn = database(‘db’, ‘root’, ‘taosdata’, ‘com.taosdata.jdbc.TSDBDriver’, ‘jdbc:TSDB://127.0.0.1:0/’)` - +```matlab +conn = database(‘db’, ‘root’, ‘taosdata’, ‘com.taosdata.jdbc.TSDBDriver’, ‘jdbc:TSDB://127.0.0.1:0/’) +``` - 执行一次查询: - `sql0 = [‘select * from tb’]` - - `data = select(conn, sql0);` - +```matlab +sql0 = [‘select * from tb’] +data = select(conn, sql0); +``` - 插入一条记录: - `sql1 = [‘insert into tb values (now, 1)’]` - - `exec(conn, sql1)` +```matlab +sql1 = [‘insert into tb values (now, 1)’] +exec(conn, sql1) +``` 更多例子细节请参考安装包内examples\Matlab\TDengineDemo.m文件。 -## R +## R R语言支持通过JDBC接口来连接TDengine数据库。首先需要安装R语言的JDBC包。启动R语言环境,然后执行以下命令安装R语言的JDBC支持库: @@ -146,4 +155,3 @@ TDengine客户端暂不支持如下函数: - dbExistsTable(conn, "test"):是否存在表test - dbListTables(conn):显示连接中的所有表 - diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/cn/10.cluster/docs.md similarity index 79% rename from documentation20/webdocs/markdowndocs/cluster-ch.md rename to documentation20/cn/10.cluster/docs.md index 89f6a64f192c65ae422c6fc52600040d439456a7..a430ce8277b49a3dbf7062fc078a47a3d848f8d8 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/cn/10.cluster/docs.md @@ -1,19 +1,19 @@ # TDengine 集群安装、管理 -多个TDengine服务器,也就是多个taosd的运行实例可以组成一个集群,以保证TDengine的高可靠运行,并提供水平扩展能力。要了解TDengine 2.0的集群管理,需要对集群的基本概念有所了解,请看TDengine 2.0整体架构一章。而且在安装集群之前,先请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验单节点功能。 +多个TDengine服务器,也就是多个taosd的运行实例可以组成一个集群,以保证TDengine的高可靠运行,并提供水平扩展能力。要了解TDengine 2.0的集群管理,需要对集群的基本概念有所了解,请看TDengine 2.0整体架构一章。而且在安装集群之前,先请按照[《立即开始》](https://www.taosdata.com/cn/documentation/getting-started/)一章安装并体验单节点功能。 集群的每个数据节点是由End Point来唯一标识的,End Point是由FQDN(Fully Qualified Domain Name)外加Port组成,比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname,可通过Linux命令`hostname -f`获取(如何配置FQDN,请参考:[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html))。端口是这个数据节点对外服务的端口号,缺省是6030,但可以通过taos.cfg里配置参数serverPort进行修改。一个物理节点可能配置了多个hostname, TDengine会自动获取第一个,但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问,可以将参数fqdn设置为本节点的IP地址。 TDengine的集群管理极其简单,除添加和删除节点需要人工干预之外,其他全部是自动完成,最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 -关于集群搭建请参考视频教程。 +关于集群搭建请参考[视频教程](https://www.taosdata.com/blog/2020/11/11/1961.html)。 -## 准备工作 +## 准备工作 **第零步**:规划集群所有物理节点的FQDN,将规划好的FQDN分别添加到每个物理节点的/etc/hostname;修改每个物理节点的/etc/hosts,将所有集群物理节点的IP与FQDN的对应添加好。【如部署了DNS,请联系网络管理员在DNS上做好相关配置】 -**第一步**:如果搭建集群的物理节点中,存有之前的测试数据、装过1.X的版本,或者装过其他版本的TDengine,请先将其删除,并清空所有数据,具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) -**注意1:**因为FQDN的信息会写进文件,如果之前没有配置或者更改FQDN,且启动了TDengine。请一定在确保数据无用或者备份的前提下,清理一下之前的数据(rm -rf /var/lib/taos/); +**第一步**:如果搭建集群的物理节点中,存有之前的测试数据、装过1.X的版本,或者装过其他版本的TDengine,请先将其删除,并清空所有数据,具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) +**注意1:**因为FQDN的信息会写进文件,如果之前没有配置或者更改FQDN,且启动了TDengine。请一定在确保数据无用或者备份的前提下,清理一下之前的数据(`rm -rf /var/lib/taos/*`); **注意2:**客户端也需要配置,确保它可以正确解析每个节点的FQDN配置,不管是通过DNS服务,还是 Host 文件。 **第二步**:建议关闭所有物理节点的防火墙,至少保证端口:6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙,集群搭建完毕之后,再来配置端口; @@ -59,15 +59,15 @@ arbitrator ha.taosdata.com:6042 | 8 | charset | 字符集编码 | | 9 | balance | 是否启动负载均衡 | | 10 | maxTablesPerVnode | 每个vnode中能够创建的最大表个数 | -| 11 | maxVgroupsPerDb | 每个DB中 能够使用的最大vnode个数 | +| 11 | maxVgroupsPerDb | 每个DB中能够使用的最大vgroup个数 | -## 启动第一个数据节点 +## 启动第一个数据节点 -按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)里的指示,启动第一个数据节点,例如h1.taosdata.com,然后执行taos, 启动taos shell,从shell里执行命令"show dnodes;",如下所示: +按照[《立即开始》](https://www.taosdata.com/cn/documentation/getting-started/)里的指示,启动第一个数据节点,例如h1.taosdata.com,然后执行taos, 启动taos shell,从shell里执行命令"show dnodes;",如下所示: - ``` +``` Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. @@ -78,15 +78,15 @@ taos> show dnodes; Query OK, 1 row(s) in set (0.006385s) taos> - ``` +``` 上述命令里,可以看到这个刚启动的这个数据节点的End Point是:h1.taos.com:6030,就是这个新集群的firstEP。 -## 启动后续数据节点 +## 启动后续数据节点 将后续的数据节点添加到现有集群,具体有以下几步: -1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法在每个物理节点启动taosd; +1. 按照[《立即开始》](https://www.taosdata.com/cn/documentation/getting-started/)一章的方法在每个物理节点启动taosd; 2. 在第一个数据节点,使用CLI程序taos, 登录进TDengine系统, 执行命令: @@ -111,11 +111,12 @@ taos> **提示:** -- 任何已经加入集群在线的数据节点,都可以作为后续待加入节点的firstEP。 -- firstEp这个参数仅仅在该数据节点首次加入集群时有作用,加入集群后,该数据节点会保存最新的mnode的End Point列表,不再依赖这个参数。 -- 两个没有配置firstEp参数的数据节点dnode启动后,会独立运行起来。这个时候,无法将其中一个数据节点加入到另外一个数据节点,形成集群。**无法将两个独立的集群合并成为新的集群**。 +- 任何已经加入集群在线的数据节点,都可以作为后续待加入节点的 firstEP。 +- firstEp 这个参数仅仅在该数据节点首次加入集群时有作用,加入集群后,该数据节点会保存最新的 mnode 的 End Point 列表,不再依赖这个参数。 + - 接下来,配置文件中的 firstEp 参数就主要在客户端连接的时候使用了,例如 taos shell 如果不加参数,会默认连接由 firstEp 指定的节点。 +- 两个没有配置 firstEp 参数的数据节点 dnode 启动后,会独立运行起来。这个时候,无法将其中一个数据节点加入到另外一个数据节点,形成集群。**无法将两个独立的集群合并成为新的集群**。 -## 数据节点管理 +## 数据节点管理 上面已经介绍如何从零开始搭建集群。集群组建完后,还可以随时添加新的数据节点进行扩容,或删除数据节点,并检查集群当前状态。 @@ -169,7 +170,7 @@ SHOW DNODES; SHOW VGROUPS; ``` -## vnode的高可用性 +## vnode的高可用性 TDengine通过多副本的机制来提供系统的高可用性,包括vnode和mnode的高可用性。 @@ -185,7 +186,7 @@ CREATE DATABASE demo replica 3; 因为vnode的引入,无法简单的给出结论:“集群中过半数据节点dnode工作,集群就应该工作”。但是对于简单的情形,很好下结论。比如副本数为3,只有三个dnode,那如果仅有一个节点不工作,整个集群还是可以正常工作的,但如果有两个数据节点不工作,那整个集群就无法正常工作了。 -## Mnode的高可用性 +## Mnode的高可用性 TDengine集群是由mnode (taosd的一个模块,管理节点) 负责管理的,为保证mnode的高可用,可以配置多个mnode副本,副本数由系统配置参数numOfMnodes决定,有效范围为1-3。为保证元数据的强一致性,mnode副本之间是通过同步的方式进行数据复制的。 @@ -202,7 +203,7 @@ SHOW MNODES; **注意:**一个TDengine高可用系统,无论是vnode还是mnode, 都必须配置多个副本。 -## 负载均衡 +## 负载均衡 有三种情况,将触发负载均衡,而且都无需人工干预。 @@ -214,18 +215,24 @@ SHOW MNODES; **【提示】负载均衡由参数balance控制,它决定是否启动自动负载均衡。** -## 数据节点离线处理 +## 数据节点离线处理 如果一个数据节点离线,TDengine集群将自动检测到。有如下两种情况: -- 该数据节点离线超过一定时间(taos.cfg里配置参数offlineThreshold控制时长),系统将自动把该数据节点删除,产生系统报警信息,触发负载均衡流程。如果该被删除的数据节点重现上线时,它将无法加入集群,需要系统管理员重新将其添加进集群才会开始工作。 +- 该数据节点离线超过一定时间(taos.cfg里配置参数offlineThreshold控制时长),系统将自动把该数据节点删除,产生系统报警信息,触发负载均衡流程。如果该被删除的数据节点重新上线时,它将无法加入集群,需要系统管理员重新将其添加进集群才会开始工作。 - 离线后,在offlineThreshold的时长内重新上线,系统将自动启动数据恢复流程,等数据完全恢复后,该节点将开始正常工作。 **注意:**如果一个虚拟节点组(包括mnode组)里所归属的每个数据节点都处于离线或unsynced状态,必须等该虚拟节点组里的所有数据节点都上线、都能交换状态信息后,才能选出Master,该虚拟节点组才能对外提供服务。比如整个集群有3个数据节点,副本数为3,如果3个数据节点都宕机,然后2个数据节点重启,是无法工作的,只有等3个数据节点都重启成功,才能对外服务。 -## Arbitrator的使用 +## Arbitrator的使用 + +如果副本数为偶数,当一个 vnode group 里一半 vnode 不工作时,是无法从中选出 master 的。同理,一半 mnode 不工作时,是无法选出 mnode 的 master 的,因为存在“split brain”问题。为解决这个问题,TDengine 引入了 Arbitrator 的概念。Arbitrator 模拟一个 vnode 或 mnode 在工作,但只简单的负责网络连接,不处理任何数据插入或访问。只要包含 Arbitrator 在内,超过半数的 vnode 或 mnode 工作,那么该 vnode group 或 mnode 组就可以正常的提供数据插入或查询服务。比如对于副本数为 2 的情形,如果一个节点 A 离线,但另外一个节点 B 正常,而且能连接到 Arbitrator,那么节点 B 就能正常工作。 -如果副本数为偶数,当一个vnode group里一半vnode不工作时,是无法从中选出master的。同理,一半mnode不工作时,是无法选出mnode的master的,因为存在“split brain”问题。为解决这个问题,TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作,但只简单的负责网络连接,不处理任何数据插入或访问。只要包含arbitrator在内,超过半数的vnode或mnode工作,那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形,如果一个节点A离线,但另外一个节点B正常,而且能连接到arbitrator, 那么节点B就能正常工作。 +总之,在目前版本下,TDengine 建议在双副本环境要配置 Arbitrator,以提升系统的可用性。 -TDengine提供一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/),在TDengine Arbitrator Linux一节中,选择适合的版本下载并安装。该程序对系统资源几乎没有要求,只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号,缺省是6042。配置每个taosd实例时,可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了,当副本数为偶数数,系统将自动连接配置的arbitrator。如果副本数为奇数,即使配置了arbitrator, 系统也不会去建立连接。 +Arbitrator 的执行程序名为 tarbitrator。该程序对系统资源几乎没有要求,只需要保证有网络连接,找任何一台 Linux 服务器运行它即可。以下简要描述安装配置的步骤: +1. 请点击 [安装包下载](https://www.taosdata.com/cn/all-downloads/),在 TDengine Arbitrator Linux 一节中,选择合适的版本下载并安装。 +2. 该应用的命令行参数 `-p` 可以指定其对外服务的端口号,缺省是 6042。 +3. 修改每个 taosd 实例的配置文件,在 taos.cfg 里将参数 arbitrator 设置为 tarbitrator 程序所对应的 End Point。(如果该参数配置了,当副本数为偶数时,系统将自动连接配置的 Arbitrator。如果副本数为奇数,即使配置了 Arbitrator,系统也不会去建立连接。) +4. 在配置文件中配置了的 Arbitrator,会出现在 `SHOW DNODES;` 指令的返回结果中,对应的 role 列的值会是“arb”。 diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/cn/11.administrator/docs.md similarity index 66% rename from documentation20/webdocs/markdowndocs/administrator-ch.md rename to documentation20/cn/11.administrator/docs.md index 9eee5b2b69cad573f62a1fa4890ff73c43026bd1..cc8689786d1725efdff82610190d4a6b1e34f906 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/cn/11.administrator/docs.md @@ -1,55 +1,65 @@ # TDengine的运营与维护 -## 容量规划 +## 容量规划 -使用TDengine来搭建一个物联网大数据平台,计算资源、存储资源需要根据业务场景进行规划。下面分别讨论系统运行所需要的内存、CPU以及硬盘空间。 +使用 TDengine 来搭建一个物联网大数据平台,计算资源、存储资源需要根据业务场景进行规划。下面分别讨论系统运行所需要的内存、CPU 以及硬盘空间。 ### 内存需求 -每个DB可以创建固定数目的vnode,默认与CPU核数相同,可通过maxVgroupsPerDb配置;每个vnode会占用固定大小的内存(大小与数据库的配置参数blocks和cache有关);每个Table会占用与标签总长度有关的内存;此外,系统会有一些固定的内存开销。因此,每个DB需要的系统内存可通过如下公式计算: +每个 Database 可以创建固定数目的 vgroup,默认与 CPU 核数相同,可通过 maxVgroupsPerDb 配置;vgroup 中的每个副本会是一个 vnode;每个 vnode 会占用固定大小的内存(大小与数据库的配置参数 blocks 和 cache 有关);每个 Table 会占用与标签总长度有关的内存;此外,系统会有一些固定的内存开销。因此,每个 DB 需要的系统内存可通过如下公式计算: ``` -Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb) +Database Memory Size = maxVgroupsPerDb * (blocks * cache + 10MB) + numOfTables * (tagSizePerTable + 0.5KB) ``` -示例:假设是4核机器,cache是缺省大小16M, blocks是缺省值6,假设有10万张表,标签总长度是256字节,则总的内存需求为:4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M。 +示例:假设是 4 核机器,cache 是缺省大小 16M, blocks 是缺省值 6,并且一个 DB 中有 10 万张表,标签总长度是 256 字节,则这个 DB 总的内存需求为:4 \* (16 \* 6 + 10) + 100000 \* (0.25 + 0.5) / 1000 = 499M。 -实际运行的系统往往会根据数据特点的不同,将数据存放在不同的DB里。因此做规划时,也需要考虑。 +在实际的系统运维中,我们通常会更关心 TDengine 服务进程(taosd)会占用的内存量。 +``` +taosd 内存总量 = vnode 内存 + mnode 内存 + 查询内存 +``` + +其中: +1. “vnode 内存”指的是集群中所有的 Database 存储分摊到当前 taosd 节点上所占用的内存资源。可以按上文“Database Memory Size”计算公式估算每个 DB 的内存占用量进行加总,再按集群中总共的 TDengine 节点数做平均(如果设置为多副本,则还需要乘以对应的副本倍数)。 +2. “mnode 内存”指的是集群中管理节点所占用的资源。如果一个 taosd 节点上分布有 mnode 管理节点,则内存消耗还需要增加“0.2KB * 集群中数据表总数”。 +3. “查询内存”指的是服务端处理查询请求时所需要占用的内存。单条查询语句至少会占用“0.2KB * 查询涉及的数据表总数”的内存量。 -如果内存充裕,可以加大Blocks的配置,这样更多数据将保存在内存里,提高查询速度。 +注意:以上内存估算方法,主要讲解了系统的“必须内存需求”,而不是“内存总数上限”。在实际运行的生产环境中,由于操作系统缓存、资源管理调度等方面的原因,内存规划应当在估算结果的基础上保留一定冗余,以维持系统状态和系统性能的稳定性。并且,生产环境通常会配置系统资源的监控工具,以便及时发现硬件资源的紧缺情况。 -### CPU需求 +最后,如果内存充裕,可以考虑加大 Blocks 的配置,这样更多数据将保存在内存里,提高查询速度。 -CPU的需求取决于如下两方面: +### CPU 需求 -* __数据插入__ TDengine单核每秒能至少处理一万个插入请求。每个插入请求可以带多条记录,一次插入一条记录与插入10条记录,消耗的计算资源差别很小。因此每次插入,条数越大,插入效率越高。如果一个插入请求带200条以上记录,单核就能达到每秒插入100万条记录的速度。但对前端数据采集的要求越高,因为需要缓存记录,然后一批插入。 -* __查询需求__ TDengine提供高效的查询,但是每个场景的查询差异很大,查询频次变化也很大,难以给出客观数字。需要用户针对自己的场景,写一些查询语句,才能确定。 +CPU 的需求取决于如下两方面: -因此仅对数据插入而言,CPU是可以估算出来的,但查询所耗的计算资源无法估算。在实际运营过程中,不建议CPU使用率超过50%,超过后,需要增加新的节点,以获得更多计算资源。 +* __数据插入__ TDengine 单核每秒能至少处理一万个插入请求。每个插入请求可以带多条记录,一次插入一条记录与插入 10 条记录,消耗的计算资源差别很小。因此每次插入,条数越大,插入效率越高。如果一个插入请求带 200 条以上记录,单核就能达到每秒插入 100 万条记录的速度。但对前端数据采集的要求越高,因为需要缓存记录,然后一批插入。 +* __查询需求__ TDengine 提供高效的查询,但是每个场景的查询差异很大,查询频次变化也很大,难以给出客观数字。需要用户针对自己的场景,写一些查询语句,才能确定。 + +因此仅对数据插入而言,CPU 是可以估算出来的,但查询所耗的计算资源无法估算。在实际运营过程中,不建议 CPU 使用率超过 50%,超过后,需要增加新的节点,以获得更多计算资源。 ### 存储需求 -TDengine相对于通用数据库,有超高的压缩比,在绝大多数场景下,TDengine的压缩比不会低于5倍,有的场合,压缩比可达到10倍以上,取决于实际场景的数据特征。压缩前的原始数据大小可通过如下方式计算: +TDengine 相对于通用数据库,有超高的压缩比,在绝大多数场景下,TDengine 的压缩比不会低于 5 倍,有的场合,压缩比可达到 10 倍以上,取决于实际场景的数据特征。压缩前的原始数据大小可通过如下方式计算: ``` Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable ``` -示例:1000万台智能电表,每台电表每15分钟采集一次数据,每次采集的数据128字节,那么一年的原始数据量是:10000000\*128\*24\*60/15*365 = 44.8512T。TDengine大概需要消耗44.851/5=8.97024T空间。 +示例:1000 万台智能电表,每台电表每 15 分钟采集一次数据,每次采集的数据 128 字节,那么一年的原始数据量是:10000000 \* 128 \* 24 \* 60 / 15 \* 365 = 44.8512T。TDengine大概需要消耗 44.851 / 5 = 8.97024T 空间。 -用户可以通过参数keep,设置数据在磁盘中的最大保存时长。为进一步减少存储成本,TDengine还提供多级存储,最冷的数据可以存放在最廉价的存储介质上,应用的访问不用做任何调整,只是读取速度降低了。 +用户可以通过参数 keep,设置数据在磁盘中的最大保存时长。为进一步减少存储成本,TDengine 还提供多级存储,最冷的数据可以存放在最廉价的存储介质上,应用的访问不用做任何调整,只是读取速度降低了。 -为提高速度,可以配置多块硬盘,这样可以并发写入或读取数据。需要提醒的是,TDengine采取多副本的方式提供数据的高可靠,因此不再需要采用昂贵的磁盘阵列。 +为提高速度,可以配置多块硬盘,这样可以并发写入或读取数据。需要提醒的是,TDengine 采取多副本的方式提供数据的高可靠,因此不再需要采用昂贵的磁盘阵列。 ### 物理机或虚拟机台数 -根据上面的内存、CPU、存储的预估,就可以知道整个系统需要多少核、多少内存、多少存储空间。如果数据副本数不为1,总需求量需要再乘以副本数。 +根据上面的内存、CPU、存储的预估,就可以知道整个系统需要多少核、多少内存、多少存储空间。如果数据副本数不为 1,总需求量需要再乘以副本数。 -因为TDengine具有很好的水平扩展能力,根据总量,再根据单个物理机或虚拟机的资源,就可以轻松决定需要购置多少台物理机或虚拟机了。 +因为 TDengine 具有很好的水平扩展能力,根据总量,再根据单个物理机或虚拟机的资源,就可以轻松决定需要购置多少台物理机或虚拟机了。 -**立即计算CPU、内存、存储,请参见:资源估算方法** +**立即计算 CPU、内存、存储,请参见:[资源估算方法](https://www.taosdata.com/config/config.html)** -## 容错和灾备 +## 容错和灾备 ### 容错 @@ -76,7 +86,7 @@ TDengine集群的节点数必须大于等于副本数,否则创建表时将报 当TDengine集群中的节点部署在不同的物理机上,并设置多个副本数时,就实现了系统的高可靠性,无需再使用其他软件或工具。TDengine企业版还可以将副本部署在不同机房,从而实现异地容灾。 -## 服务端配置 +## 服务端配置 TDengine系统后台服务由taosd提供,可以在配置文件taos.cfg里修改配置参数,以满足不同场景的需求。配置文件的缺省位置在/etc/taos目录,可以通过taosd命令行执行参数-c指定配置文件目录。比如taosd -c /home/user来指定配置文件位于/home/user这个目录。 @@ -102,7 +112,7 @@ taosd -C - maxSQLLength:单条SQL语句允许最长限制。默认值:65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息,0表示不允许,1表示允许。 默认值:1。 - stream: 是否启用连续查询(流计算功能),0表示不允许,1表示允许。 默认值:1。 -- queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘,再乘 170 。单位为字节。 +- queryBufferSize: 为所有并发查询占用保留的内存大小。计算规则可以根据实际应用可能的最大并发数和表的数字相乘,再乘 170 。单位为 MB(2.0.15 以前的版本中,此参数的单位是字节)。 - ratioOfQueryCores: 设置查询线程的最大数量。最小值0 表示只有1个查询线程;最大值2表示最大建立2倍CPU核数的查询线程。默认为1,表示最大和CPU核数相等的查询线程。该值可以为小数,即0.5表示最大建立CPU核数一半的查询线程。 **注意:**对于端口,TDengine会使用从serverPort起13个连续的TCP和UDP端口号,请务必在防火墙打开。因此如果是缺省配置,需要打开从6030到6042共13个端口,而且必须TCP和UDP都打开。 @@ -110,24 +120,25 @@ taosd -C 不同应用场景的数据往往具有不同的数据特征,比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率,TDengine提供如下存储相关的系统配置参数: - days:一个数据文件存储数据的时间跨度,单位为天,默认值:10。 -- keep:数据库中数据保留的天数,单位为天,默认值:3650。 -- minRows: 文件块中记录的最小条数,单位为条,默认值:100。 -- maxRows: 文件块中记录的最大条数,单位为条,默认值:4096。 -- comp: 文件压缩标志位,0:关闭,1:一阶段压缩,2:两阶段压缩。默认值:2。 -- walLevel:WAL级别。1:写wal, 但不执行fsync; 2:写wal, 而且执行fsync。默认值:1。 +- keep:数据库中数据保留的天数,单位为天,默认值:3650。(可通过 alter database 修改) +- minRows:文件块中记录的最小条数,单位为条,默认值:100。 +- maxRows:文件块中记录的最大条数,单位为条,默认值:4096。 +- comp:文件压缩标志位,0:关闭;1:一阶段压缩;2:两阶段压缩。默认值:2。(可通过 alter database 修改) +- walLevel:WAL级别。1:写wal,但不执行fsync;2:写wal, 而且执行fsync。默认值:1。 - fsync:当wal设置为2时,执行fsync的周期。设置为0,表示每次写入,立即执行fsync。单位为毫秒,默认值:3000。 -- cache: 内存块的大小,单位为兆字节(MB),默认值:16。 -- blocks: 每个VNODE(TSDB)中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为(cache * blocks)。单位为块,默认值:4。 -- replica:副本个数,取值范围:1-3。单位为个,默认值:1 -- precision:时间戳精度标识,ms表示毫秒,us表示微秒。默认值:ms +- cache:内存块的大小,单位为兆字节(MB),默认值:16。 +- blocks:每个VNODE(TSDB)中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为(cache * blocks)。单位为块,默认值:4。(可通过 alter database 修改) +- replica:副本个数,取值范围:1-3。单位为个,默认值:1。(可通过 alter database 修改) +- precision:时间戳精度标识,ms表示毫秒,us表示微秒。默认值:ms。 +- cacheLast:是否在内存中缓存子表 last_row,0:关闭;1:开启。默认值:0。(可通过 alter database 修改)(从 2.0.11 版本开始支持此参数) 对于一个应用场景,可能有多种数据特征的数据并存,最佳的设计是将具有相同数据特征的表放在一个库里,这样一个应用有多个库,而每个库可以配置不同的存储参数,从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数,如果指定,该参数就将覆盖对应的系统配置参数。举例,有下述SQL: ``` - create database demo days 10 cache 32 blocks 8 replica 3; + create database demo days 10 cache 32 blocks 8 replica 3 update 1; ``` -该SQL创建了一个库demo, 每个数据文件存储10天数据,内存块为32兆字节,每个VNODE占用8个内存块,副本数为3,而其他参数与系统配置完全一致。 +该SQL创建了一个库demo, 每个数据文件存储10天数据,内存块为32兆字节,每个VNODE占用8个内存块,副本数为3,允许更新,而其他参数与系统配置完全一致。 TDengine集群中加入一个新的dnode时,涉及集群相关的一些参数必须与已有集群的配置相同,否则不能成功加入到集群中。会进行校验的参数如下: @@ -137,7 +148,7 @@ TDengine集群中加入一个新的dnode时,涉及集群相关的一些参数 - offlineThreshold: dnode离线阈值,超过该时间将导致该dnode从集群中删除。单位为秒,默认值:86400*10(即10天)。 - statusInterval: dnode向mnode报告状态时长。单位为秒,默认值:1。 - maxTablesPerVnode: 每个vnode中能够创建的最大表个数。默认值:1000000。 -- maxVgroupsPerDb: 每个数据库中能够使用的最大vnode个数。 +- maxVgroupsPerDb: 每个数据库中能够使用的最大vgroup个数。 - arbitrator: 系统中裁决器的end point,缺省为空。 - timezone、locale、charset 的配置见客户端配置。 @@ -157,9 +168,9 @@ ALTER DNODE alter dnode 1 debugFlag 135; ``` -## 客户端配置 +## 客户端配置 -TDengine系统的前台交互客户端应用程序为taos,以及应用驱动,它与taosd共享同一个配置文件taos.cfg。运行taos时,使用参数-c指定配置文件目录,如taos -c /home/cfg,表示使用/home/cfg/目录下的taos.cfg配置文件中的参数,缺省目录是/etc/taos。更多taos的使用方法请见Shell命令行程序。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。 +TDengine系统的前台交互客户端应用程序为taos,以及应用驱动,它与taosd共享同一个配置文件taos.cfg。运行taos时,使用参数-c指定配置文件目录,如taos -c /home/cfg,表示使用/home/cfg/目录下的taos.cfg配置文件中的参数,缺省目录是/etc/taos。更多taos的使用方法请见帮助信息 `taos --help`。本节主要说明 taos 客户端应用在配置文件 taos.cfg 文件中使用到的参数。 **2.0.10.0 之后版本支持命令行以下参数显示当前客户端参数的配置** @@ -181,7 +192,7 @@ taos -C 或 taos --dump-config 客户端的输入的字符均采用操作系统当前默认的编码格式,在Linux系统上多为UTF-8,部分中文系统编码则可能是GB18030或GBK等。在docker环境中默认的编码是POSIX。在中文版Windows系统中,编码则是CP936。客户端需要确保正确设置自己所使用的字符集,即客户端运行的操作系统当前编码字符集,才能保证nchar中的数据正确转换为UCS4-LE编码格式。 - 在 Linux 中 locale 的命名规则为: <语言>_<地区>.<字符集编码> 如:zh_CN.UTF-8,zh代表中文,CN代表大陆地区,UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码,由于Windows使用的locale中不是POSIX标准的locale格式,因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。 + 在 Linux 中 locale 的命名规则为: <语言>\_<地区>.<字符集编码> 如:zh_CN.UTF-8,zh代表中文,CN代表大陆地区,UTF-8表示字符集。字符集编码为客户端正确解析本地字符串提供编码转换的说明。Linux系统与 Mac OSX 系统可以通过设置locale来确定系统的字符编码,由于Windows使用的locale中不是POSIX标准的locale格式,因此在Windows下需要采用另一个配置参数charset来指定字符编码。在Linux 系统中也可以使用charset来指定字符编码。 - charset @@ -247,7 +258,7 @@ taos -C 或 taos --dump-config Shell中binary 和 nchar字段的显示宽度上限,超过此限制的部分将被隐藏。默认值:30。可在 shell 中通过命令 set max_binary_display_width nn 动态修改此选项。 -## 用户管理 +## 用户管理 系统管理员可以在CLI界面里添加、删除用户,也可以修改密码。CLI里SQL语法如下: @@ -285,7 +296,7 @@ SHOW USERS; **注意:**SQL 语法中,< >表示需要用户输入的部分,但请不要输入< >本身 -## 数据导入 +## 数据导入 TDengine提供多种方便的数据导入功能,一种按脚本文件导入,一种按数据文件导入,一种是taosdump工具导入本身导出的文件。 @@ -337,9 +348,9 @@ Query OK, 9 row(s) affected (0.004763s) **taosdump工具导入** -TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据,导入到其他系统中。具体使用方法,请参见博客:TDengine DUMP工具使用指南 +TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据,导入到其他系统中。具体使用方法,请参见博客:[TDengine DUMP工具使用指南](https://www.taosdata.com/blog/2020/03/09/1334.html) -## 数据导出 +## 数据导出 为方便数据导出,TDengine提供了两种导出方式,分别是按表导出和用taosdump导出。 @@ -355,9 +366,9 @@ select * from >> data.csv; **用taosdump导出数据** -TDengine提供了方便的数据库导出工具taosdump。用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表,所有数据或一时间段的数据,甚至仅仅表的定义。具体使用方法,请参见博客:TDengine DUMP工具使用指南 +TDengine提供了方便的数据库导出工具taosdump。用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表,所有数据或一时间段的数据,甚至仅仅表的定义。具体使用方法,请参见博客:[TDengine DUMP工具使用指南](https://www.taosdata.com/blog/2020/03/09/1334.html) -## 系统连接、任务查询管理 +## 系统连接、任务查询管理 系统管理员可以从CLI查询系统的连接、正在进行的查询、流式计算,并且可以关闭连接、停止正在进行的查询和流式计算。CLI里SQL语法如下: @@ -403,7 +414,7 @@ TDengine启动后,会自动创建一个监测数据库log,并自动将服务 这些监测信息的采集缺省是打开的,但可以修改配置文件里的选项enableMonitor将其关闭或打开。 -## 文件目录结构 +## 文件目录结构 安装TDengine后,默认会在操作系统中生成下列目录或文件: @@ -429,62 +440,64 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。 -## TDengine参数限制与保留关键字 +## TDengine参数限制与保留关键字 -- 数据库名:不能包含“.”以及特殊字符,不能超过32个字符 -- 表名:不能包含“.”以及特殊字符,与所属数据库名一起,不能超过192个字符 -- 表的列名:不能包含特殊字符,不能超过64个字符 +- 数据库名:不能包含“.”以及特殊字符,不能超过 32 个字符 +- 表名:不能包含“.”以及特殊字符,与所属数据库名一起,不能超过 192 个字符 +- 表的列名:不能包含特殊字符,不能超过 64 个字符 - 数据库名、表名、列名,都不能以数字开头 -- 表的列数:不能超过1024列 -- 记录的最大长度:包括时间戳8 byte,不能超过16KB -- 单条SQL语句默认最大字符串长度:65480 byte -- 数据库副本数:不能超过3 -- 用户名:不能超过23个byte -- 用户密码:不能超过15个byte -- 标签(Tags)数量:不能超过128个 -- 标签的总长度:不能超过16Kbyte +- 表的列数:不能超过 1024 列 +- 记录的最大长度:包括时间戳 8 byte,不能超过 16KB(每个 BINARY/NCHAR 类型的列还会额外占用 2 个 byte 的存储位置) +- 单条 SQL 语句默认最大字符串长度:65480 byte +- 数据库副本数:不能超过 3 +- 用户名:不能超过 23 个 byte +- 用户密码:不能超过 15 个 byte +- 标签(Tags)数量:不能超过 128 个 +- 标签的总长度:不能超过 16K byte - 记录条数:仅受存储空间限制 - 表的个数:仅受节点个数限制 - 库的个数:仅受节点个数限制 -- 单个库上虚拟节点个数:不能超过64个 +- 单个库上虚拟节点个数:不能超过 64 个 -目前TDengine有将近200个内部保留关键字,这些关键字无论大小写均不可以用作库名、表名、STable名、数据列名及标签列名等。这些关键字列表如下: +目前 TDengine 有将近 200 个内部保留关键字,这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下: | 关键字列表 | | | | | | ---------- | ----------- | ------------ | ---------- | --------- | -| ABLOCKS | CONNECTION | GT | MINUS | SHOW | -| ABORT | CONNECTIONS | ID | MNODES | SLASH | -| ACCOUNT | COPY | IF | MODULES | SLIDING | -| ACCOUNTS | COUNT | IGNORE | NCHAR | SMALLINT | -| ADD | CREATE | IMMEDIATE | NE | SPREAD | -| AFTER | CTIME | IMPORT | NONE | STAR | -| ALL | DATABASE | IN | NOT | STATEMENT | -| ALTER | DATABASES | INITIALLY | NOTNULL | STDDEV | -| AND | DAYS | INSERT | NOW | STREAM | -| AS | DEFERRED | INSTEAD | OF | STREAMS | -| ASC | DELIMITERS | INTEGER | OFFSET | STRING | -| ATTACH | DESC | INTERVAL | OR | SUM | -| AVG | DESCRIBE | INTO | ORDER | TABLE | -| BEFORE | DETACH | IP | PASS | TABLES | -| BEGIN | DIFF | IS | PERCENTILE | TAG | -| BETWEEN | DIVIDE | ISNULL | PLUS | TAGS | -| BIGINT | DNODE | JOIN | PRAGMA | TBLOCKS | -| BINARY | DNODES | KEEP | PREV | TBNAME | -| BITAND | DOT | KEY | PRIVILEGE | TIMES | -| BITNOT | DOUBLE | KILL | QUERIES | TIMESTAMP | -| BITOR | DROP | LAST | QUERY | TINYINT | -| BOOL | EACH | LE | RAISE | TOP | -| BOTTOM | END | LEASTSQUARES | REM | TRIGGER | -| BY | EQ | LIKE | REPLACE | UMINUS | -| CACHE | EXISTS | LIMIT | REPLICA | UPLUS | -| CASCADE | EXPLAIN | LINEAR | RESET | USE | -| CHANGE | FAIL | LOCAL | RESTRICT | USER | -| CLOG | FILL | LP | ROW | USERS | -| CLUSTER | FIRST | LSHIFT | ROWS | USING | -| COLON | FLOAT | LT | RP | VALUES | -| COLUMN | FOR | MATCH | RSHIFT | VARIABLE | -| COMMA | FROM | MAX | SCORES | VGROUPS | -| COMP | GE | METRIC | SELECT | VIEW | -| CONCAT | GLOB | METRICS | SEMI | WAVG | -| CONFIGS | GRANTS | MIN | SET | WHERE | -| CONFLICT | GROUP | | | | \ No newline at end of file +| ABLOCKS | CONNECTIONS | GT | MNODES | SLIDING | +| ABORT | COPY | ID | MODULES | SLIMIT | +| ACCOUNT | COUNT | IF | NCHAR | SMALLINT | +| ACCOUNTS | CREATE | IGNORE | NE | SPREAD | +| ADD | CTIME | IMMEDIATE | NONE | STABLE | +| AFTER | DATABASE | IMPORT | NOT | STABLES | +| ALL | DATABASES | IN | NOTNULL | STAR | +| ALTER | DAYS | INITIALLY | NOW | STATEMENT | +| AND | DEFERRED | INSERT | OF | STDDEV | +| AS | DELIMITERS | INSTEAD | OFFSET | STREAM | +| ASC | DESC | INTEGER | OR | STREAMS | +| ATTACH | DESCRIBE | INTERVAL | ORDER | STRING | +| AVG | DETACH | INTO | PASS | SUM | +| BEFORE | DIFF | IP | PERCENTILE | TABLE | +| BEGIN | DISTINCT | IS | PLUS | TABLES | +| BETWEEN | DIVIDE | ISNULL | PRAGMA | TAG | +| BIGINT | DNODE | JOIN | PREV | TAGS | +| BINARY | DNODES | KEEP | PRIVILEGE | TBLOCKS | +| BITAND | DOT | KEY | QUERIES | TBNAME | +| BITNOT | DOUBLE | KILL | QUERY | TIMES | +| BITOR | DROP | LAST | RAISE | TIMESTAMP | +| BOOL | EACH | LE | REM | TINYINT | +| BOTTOM | END | LEASTSQUARES | REPLACE | TOP | +| BY | EQ | LIKE | REPLICA | TRIGGER | +| CACHE | EXISTS | LIMIT | RESET | UMINUS | +| CASCADE | EXPLAIN | LINEAR | RESTRICT | UPLUS | +| CHANGE | FAIL | LOCAL | ROW | USE | +| CLOG | FILL | LP | ROWS | USER | +| CLUSTER | FIRST | LSHIFT | RP | USERS | +| COLON | FLOAT | LT | RSHIFT | USING | +| COLUMN | FOR | MATCH | SCORES | VALUES | +| COMMA | FROM | MAX | SELECT | VARIABLE | +| COMP | GE | METRIC | SEMI | VGROUPS | +| CONCAT | GLOB | METRICS | SET | VIEW | +| CONFIGS | GRANTS | MIN | SHOW | WAVG | +| CONFLICT | GROUP | MINUS | SLASH | WHERE | +| CONNECTION | | | | | + diff --git a/documentation20/webdocs/markdowndocs/Taos Error Code-ch.md b/documentation20/cn/12.taos-sql/01.error-code/docs.md similarity index 100% rename from documentation20/webdocs/markdowndocs/Taos Error Code-ch.md rename to documentation20/cn/12.taos-sql/01.error-code/docs.md diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/cn/12.taos-sql/docs.md similarity index 72% rename from documentation20/webdocs/markdowndocs/TAOS SQL-ch.md rename to documentation20/cn/12.taos-sql/docs.md index bee3d99b2630e53c7a2154f0f605077bcc56b267..58191e0bd8faa02b0ff2381f1cdd576b379ae9fc 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -1,17 +1,19 @@ # TAOS SQL -本文档说明TAOS SQL支持的语法规则、主要查询功能、支持的SQL查询函数,以及常用技巧等内容。阅读本文档需要读者具有基本的SQL语言的基础。 +本文档说明 TAOS SQL 支持的语法规则、主要查询功能、支持的 SQL 查询函数,以及常用技巧等内容。阅读本文档需要读者具有基本的 SQL 语言的基础。 -TAOS SQL是用户对TDengine进行数据写入和查询的主要工具。TAOS SQL为了便于用户快速上手,在一定程度上提供类似于标准SQL类似的风格和模式。严格意义上,TAOS SQL并不是也不试图提供SQL标准的语法。此外,由于TDengine针对的时序性结构化数据不提供删除功能,因此在TAO SQL中不提供数据删除的相关功能。 +TAOS SQL 是用户对 TDengine 进行数据写入和查询的主要工具。TAOS SQL 为了便于用户快速上手,在一定程度上提供类似于标准 SQL 类似的风格和模式。严格意义上,TAOS SQL 并不是也不试图提供 SQL 标准的语法。此外,由于 TDengine 针对的时序性结构化数据不提供删除功能,因此在 TAO SQL 中不提供数据删除的相关功能。 -本章节SQL语法遵循如下约定: +TAOS SQL 不支持关键字的缩写,例如 DESCRIBE 不能缩写为 DESC。 -- < > 里的内容是用户需要输入的,但不要输入<>本身 -- [ ]表示内容为可选项,但不能输入[]本身 -- | 表示多选一,选择其中一个即可,但不能输入|本身 +本章节 SQL 语法遵循如下约定: + +- < > 里的内容是用户需要输入的,但不要输入 <> 本身 +- [ ] 表示内容为可选项,但不能输入 [] 本身 +- | 表示多选一,选择其中一个即可,但不能输入 | 本身 - … 表示前面的项可重复多个 -为更好地说明SQL语法的规则及其特点,本文假设存在一个数据集。以智能电表(meters)为例,假设每个智能电表采集电流、电压、相位三个量。其建模如下: +为更好地说明 SQL 语法的规则及其特点,本文假设存在一个数据集。以智能电表(meters)为例,假设每个智能电表采集电流、电压、相位三个量。其建模如下: ```mysql taos> DESCRIBE meters; Field | Type | Length | Note | @@ -23,61 +25,63 @@ taos> DESCRIBE meters; location | BINARY | 64 | TAG | groupid | INT | 4 | TAG | ``` -数据集包含4个智能电表的数据,按照TDengine的建模规则,对应4个子表,其名称分别是 d1001, d1002, d1003, d1004。 +数据集包含 4 个智能电表的数据,按照 TDengine 的建模规则,对应 4 个子表,其名称分别是 d1001, d1002, d1003, d1004。 -## 支持的数据类型 +## 支持的数据类型 -使用TDengine,最重要的是时间戳。创建并插入记录、查询历史记录的时候,均需要指定时间戳。时间戳有如下规则: +使用 TDengine,最重要的是时间戳。创建并插入记录、查询历史记录的时候,均需要指定时间戳。时间戳有如下规则: -- 时间格式为```YYYY-MM-DD HH:mm:ss.MS```, 默认时间分辨率为毫秒。比如:```2017-08-12 18:25:58.128``` -- 内部函数now是服务器的当前时间 -- 插入记录时,如果时间戳为now,插入数据时使用服务器当前时间 -- Epoch Time: 时间戳也可以是一个长整数,表示从1970-01-01 08:00:00.000开始的毫秒数 -- 时间可以加减,比如 now-2h,表明查询时刻向前推2个小时(最近2小时)。 数字后面的时间单位可以是 a(毫秒)、s(秒)、 m(分)、h(小时)、d(天)、w(周)。 比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据。 在指定降频操作(down sampling)的时间窗口(interval)时,时间单位还可以使用 n(自然月) 和 y(自然年)。 +- 时间格式为 ```YYYY-MM-DD HH:mm:ss.MS```,默认时间分辨率为毫秒。比如:```2017-08-12 18:25:58.128``` +- 内部函数 now 是客户端的当前时间 +- 插入记录时,如果时间戳为 now,插入数据时使用提交这条记录的客户端的当前时间 +- Epoch Time:时间戳也可以是一个长整数,表示从 1970-01-01 08:00:00.000 开始的毫秒数 +- 时间可以加减,比如 now-2h,表明查询时刻向前推 2 个小时(最近 2 小时)。数字后面的时间单位可以是 u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`,表示查询两周前整整一周的数据。在指定降频操作(down sampling)的时间窗口(interval)时,时间单位还可以使用 n(自然月) 和 y(自然年)。 -TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMicrosecond就可支持微秒。 +TDengine 缺省的时间戳是毫秒精度,但通过修改配置参数 enableMicrosecond 就可以支持微秒。 -在TDengine中,普通表的数据模型中可使用以下10种数据类型。 +在TDengine中,普通表的数据模型中可使用以下 10 种数据类型。 | | 类型 | Bytes | 说明 | | ---- | :-------: | ------ | ------------------------------------------------------------ | -| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒,可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始,计时不能早于该时间。 | -| 2 | INT | 4 | 整型,范围 [-2^31+1, 2^31-1], -2^31用作Null | -| 3 | BIGINT | 8 | 长整型,范围 [-2^63+1, 2^63-1], -2^63用于NULL | -| 4 | FLOAT | 4 | 浮点型,有效位数6-7,范围 [-3.4E38, 3.4E38] | -| 5 | DOUBLE | 8 | 双精度浮点型,有效位数15-16,范围 [-1.7E308, 1.7E308] | -| 6 | BINARY | 自定义 | 用于记录字符串,理论上,最长可以有16374字节,但由于每行数据最多16K字节,实际上限一般小于理论值。 binary仅支持字符串输入,字符串两端使用单引号引用,否则英文全部自动转化为小写。使用时须指定大小,如binary(20)定义了最长为20个字符的字符串,每个字符占1byte的存储空间。如果用户字符串超出20字节将会报错。对于字符串内的单引号,可以用转义字符反斜线加单引号来表示, 即 **\’**。 | -| 7 | SMALLINT | 2 | 短整型, 范围 [-32767, 32767], -32768用于NULL | -| 8 | TINYINT | 1 | 单字节整型,范围 [-127, 127], -128用于NULL | -| 9 | BOOL | 1 | 布尔型,{true, false} | -| 10 | NCHAR | 自定义 | 用于记录非ASCII字符串,如中文字符。每个nchar字符占用4bytes的存储空间。字符串两端使用单引号引用,字符串内的单引号需用转义字符 **\’**。nchar使用时须指定字符串大小,类型为nchar(10)的列表示此列的字符串最多存储10个nchar字符,会固定占用40bytes的空间。如用户字符串长度超出声明长度,则将会报错。 | - -**Tips**: TDengine对SQL语句中的英文字符不区分大小写,自动转化为小写执行。因此用户大小写敏感的字符串及密码,需要使用单引号将字符串引起来。 - -## 数据库管理 +| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒,可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始,计时不能早于该时间。(从 2.0.18 版本开始,已经去除了这一时间范围限制) | +| 2 | INT | 4 | 整型,范围 [-2^31+1, 2^31-1], -2^31 用作 NULL | +| 3 | BIGINT | 8 | 长整型,范围 [-2^63+1, 2^63-1], -2^63 用于 NULL | +| 4 | FLOAT | 4 | 浮点型,有效位数 6-7,范围 [-3.4E38, 3.4E38] | +| 5 | DOUBLE | 8 | 双精度浮点型,有效位数 15-16,范围 [-1.7E308, 1.7E308] | +| 6 | BINARY | 自定义 | 用于记录 ASCII 型字符串。理论上,最长可以有 16374 字节,但由于每行数据最多 16K 字节,实际上限一般小于理论值。 binary 仅支持字符串输入,字符串两端使用单引号引用,否则英文全部自动转化为小写。使用时须指定大小,如 binary(20) 定义了最长为 20 个字符的字符串,每个字符占 1 byte 的存储空间,此时如果用户字符串超出 20 字节将会报错。对于字符串内的单引号,可以用转义字符反斜线加单引号来表示,即 `\’`。 | +| 7 | SMALLINT | 2 | 短整型, 范围 [-32767, 32767], -32768 用于 NULL | +| 8 | TINYINT | 1 | 单字节整型,范围 [-127, 127], -128 用于 NULL | +| 9 | BOOL | 1 | 布尔型,{true, false} | +| 10 | NCHAR | 自定义 | 用于记录非 ASCII 型字符串,如中文字符。每个 nchar 字符占用 4 bytes 的存储空间。字符串两端使用单引号引用,字符串内的单引号需用转义字符 `\’`。nchar 使用时须指定字符串大小,类型为 nchar(10) 的列表示此列的字符串最多存储 10 个 nchar 字符,会固定占用 40 bytes 的空间。如果用户字符串长度超出声明长度,将会报错。 | + +**Tips**: +1. TDengine 对 SQL 语句中的英文字符不区分大小写,自动转化为小写执行。因此用户大小写敏感的字符串及密码,需要使用单引号将字符串引起来。 +2. 应避免使用 BINARY 类型来保存非 ASCII 型的字符串,会很容易导致数据乱码等错误。正确的做法是使用 NCHAR 类型来保存中文字符。 + +## 数据库管理 - **创建数据库** - ```mysql - CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [UPDATE 1]; - ``` - 说明: - - 1) KEEP是该数据库的数据保留多长天数,缺省是3650天(10年),数据库会自动删除超过时限的数据; - - 2) UPDATE 标志数据库支持更新相同时间戳数据; - - 3) 数据库名最大长度为33; - - 4) 一条SQL 语句的最大长度为65480个字符; - - 5) 数据库还有更多与存储相关的配置参数,请参见系统管理。 + ```mysql + CREATE DATABASE [IF NOT EXISTS] db_name [KEEP keep] [UPDATE 1]; + ``` + 说明: + + 1) KEEP是该数据库的数据保留多长天数,缺省是3650天(10年),数据库会自动删除超过时限的数据; + + 2) UPDATE 标志数据库支持更新相同时间戳数据; + + 3) 数据库名最大长度为33; + + 4) 一条SQL 语句的最大长度为65480个字符; + + 5) 数据库还有更多与存储相关的配置参数,请参见系统管理。 - **显示系统当前参数** - ```mysql - SHOW VARIABLES; - ``` + ```mysql + SHOW VARIABLES; + ``` - **使用数据库** @@ -96,36 +100,43 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic ```mysql ALTER DATABASE db_name COMP 2; ``` - COMP参数是指修改数据库文件压缩标志位,取值范围为[0, 2]. 0表示不压缩,1表示一阶段压缩,2表示两阶段压缩。 + COMP 参数是指修改数据库文件压缩标志位,缺省值为 2,取值范围为 [0, 2]。0 表示不压缩,1 表示一阶段压缩,2 表示两阶段压缩。 ```mysql ALTER DATABASE db_name REPLICA 2; ``` - REPLICA参数是指修改数据库副本数,取值范围[1, 3]。在集群中使用,副本数必须小于或等于dnode的数目。 + REPLICA 参数是指修改数据库副本数,取值范围 [1, 3]。在集群中使用,副本数必须小于或等于 DNODE 的数目。 ```mysql ALTER DATABASE db_name KEEP 365; ``` - KEEP参数是指修改数据文件保存的天数,缺省值为3650,取值范围[days, 365000],必须大于或等于days参数值。 + KEEP 参数是指修改数据文件保存的天数,缺省值为 3650,取值范围 [days, 365000],必须大于或等于 days 参数值。 ```mysql ALTER DATABASE db_name QUORUM 2; ``` - QUORUM参数是指数据写入成功所需要的确认数。取值范围[1, 3]。对于异步复制,quorum设为1,具有master角色的虚拟节点自己确认即可。对于同步复制,需要至少大于等于2。原则上,Quorum >=1 并且 Quorum <= replica(副本数),这个参数在启动一个同步模块实例时需要提供。 + QUORUM 参数是指数据写入成功所需要的确认数,取值范围 [1, 2]。对于异步复制,quorum 设为 1,具有 master 角色的虚拟节点自己确认即可。对于同步复制,需要至少大于等于 2。原则上,Quorum >= 1 并且 Quorum <= replica(副本数),这个参数在启动一个同步模块实例时需要提供。 ```mysql ALTER DATABASE db_name BLOCKS 100; ``` - BLOCKS参数是每个VNODE (TSDB) 中有多少cache大小的内存块,因此一个VNODE的用的内存大小粗略为(cache * blocks)。取值范围[3, 1000]。 + BLOCKS 参数是每个 VNODE (TSDB) 中有多少 cache 大小的内存块,因此一个 VNODE 的用的内存大小粗略为(cache * blocks)。取值范围 [3, 1000]。 + + ```mysql + ALTER DATABASE db_name CACHELAST 0; + ``` + CACHELAST 参数控制是否在内存中缓存数据子表的 last_row。缺省值为 0,取值范围 [0, 1]。其中 0 表示不启用、1 表示启用。(从 2.0.11 版本开始支持,修改后需要重启服务器生效。) **Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。 - **显示系统所有数据库** + ```mysql SHOW DATABASES; ``` -## 表管理 +## 表管理 + - **创建数据表** ```mysql @@ -133,15 +144,15 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic ``` 说明: - 1) 表的第一个字段必须是TIMESTAMP,并且系统自动将其设为主键; + 1) 表的第一个字段必须是 TIMESTAMP,并且系统自动将其设为主键; - 2) 表名最大长度为192; + 2) 表名最大长度为 192; - 3) 表的每行长度不能超过16k个字符; + 3) 表的每行长度不能超过 16k 个字符;(注意:每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置) 4) 子表名只能由字母、数字和下划线组成,且不能以数字开头 - 5) 使用数据类型binary或nchar,需指定其最长的字节数,如binary(20),表示20字节; + 5) 使用数据类型 binary 或 nchar,需指定其最长的字节数,如 binary(20),表示 20 字节; - **以超级表为模板创建数据表** @@ -150,6 +161,14 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic ``` 以指定的超级表为模板,指定 tags 的值来创建数据表。 +- **以超级表为模板创建数据表,并指定具体的 tags 列** + + ```mysql + CREATE TABLE [IF NOT EXISTS] tb_name USING stb_name (tag_name1, ...) TAGS (tag_value1, ...); + ``` + 以指定的超级表为模板,指定一部分 tags 列的值来创建数据表。(没被指定的 tags 列会设为空值。) + 说明:从 2.0.17 版本开始支持这种方式。在之前的版本中,不允许指定 tags 列,而必须显式给出所有 tags 列的取值。 + - **批量创建数据表** ```mysql @@ -211,37 +230,40 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic ``` 如果表是通过[超级表](../super-table/)创建,更改表结构的操作只能对超级表进行。同时针对超级表的结构更改对所有通过该结构创建的表生效。对于不是通过超级表创建的表,可以直接修改表结构 -## 超级表STable管理 +## 超级表STable管理 + +注意:在 2.0.15.0 及以后的版本中,开始支持 STABLE 保留字。也即,在本节后文的指令说明中,CREATE、DROP、ALTER 三个指令在老版本中保留字需写作 TABLE 而不是 STABLE。 + - **创建超级表** ```mysql - CREATE TABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); + CREATE STABLE [IF NOT EXISTS] stb_name (timestamp_field_name TIMESTAMP, field1_name data_type1 [, field2_name data_type2 ...]) TAGS (tag1_name tag_type1, tag2_name tag_type2 [, tag3_name tag_type3]); ``` - 创建STable, 与创建表的SQL语法相似,但需指定TAGS字段的名称和类型 + 创建 STable,与创建表的 SQL 语法相似,但需指定 TAGS 字段的名称和类型 说明: - 1) TAGS 列的数据类型不能是timestamp类型; + 1) TAGS 列的数据类型不能是 timestamp 类型; 2) TAGS 列名不能与其他列名相同; 3) TAGS 列名不能为预留关键字; - 4) TAGS 最多允许128个,至少1个,总长度不超过16k个字符。 + 4) TAGS 最多允许 128 个,至少 1 个,总长度不超过 16 KB。 - **删除超级表** ```mysql - DROP TABLE [IF EXISTS] stb_name; + DROP STABLE [IF EXISTS] stb_name; ``` - 删除STable会自动删除通过STable创建的子表。 + 删除 STable 会自动删除通过 STable 创建的子表。 - **显示当前数据库下的所有超级表信息** ```mysql SHOW STABLES [LIKE tb_name_wildcar]; ``` - 查看数据库内全部STable,及其相关信息,包括STable的名称、创建时间、列数量、标签(TAG)数量、通过该STable建表的数量。 + 查看数据库内全部 STable,及其相关信息,包括 STable 的名称、创建时间、列数量、标签(TAG)数量、通过该 STable 建表的数量。 - **获取超级表的结构信息** @@ -252,34 +274,35 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic - **超级表增加列** ```mysql - ALTER TABLE stb_name ADD COLUMN field_name data_type; + ALTER STABLE stb_name ADD COLUMN field_name data_type; ``` - **超级表删除列** ```mysql - ALTER TABLE stb_name DROP COLUMN field_name; + ALTER STABLE stb_name DROP COLUMN field_name; ``` -## 超级表 STable 中 TAG 管理 +## 超级表 STable 中 TAG 管理 + - **添加标签** ```mysql - ALTER TABLE stb_name ADD TAG new_tag_name tag_type; + ALTER STABLE stb_name ADD TAG new_tag_name tag_type; ``` - 为STable增加一个新的标签,并指定新标签的类型。标签总数不能超过128个,总长度不超过16k个字符。 + 为 STable 增加一个新的标签,并指定新标签的类型。标签总数不能超过 128 个,总长度不超过 16k 个字符。 - **删除标签** ```mysql - ALTER TABLE stb_name DROP TAG tag_name; + ALTER STABLE stb_name DROP TAG tag_name; ``` 删除超级表的一个标签,从超级表删除某个标签后,该超级表下的所有子表也会自动删除该标签。 - **修改标签名** ```mysql - ALTER TABLE stb_name CHANGE TAG old_tag_name new_tag_name; + ALTER STABLE stb_name CHANGE TAG old_tag_name new_tag_name; ``` 修改超级表的标签名,从超级表修改某个标签名后,该超级表下的所有子表也会自动更新该标签名。 @@ -288,9 +311,9 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic ```mysql ALTER TABLE tb_name SET TAG tag_name=new_tag_value; ``` - 说明:除了更新标签的值的操作是针对子表进行,其他所有的标签操作(添加标签、删除标签等)均只能作用于STable,不能对单个子表操作。对STable添加标签以后,依托于该STable建立的所有表将自动增加了一个标签,所有新增标签的默认值都是NULL。 + 说明:除了更新标签的值的操作是针对子表进行,其他所有的标签操作(添加标签、删除标签等)均只能作用于 STable,不能对单个子表操作。对 STable 添加标签以后,依托于该 STable 建立的所有表将自动增加了一个标签,所有新增标签的默认值都是 NULL。 -## 数据写入 +## 数据写入 - **插入一条记录** ```mysql @@ -300,7 +323,7 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic - **插入一条记录,数据对应到指定的列** ```mysql - INSERT INTO tb_name (field1_name, ...) VALUES (field1_value, ...) + INSERT INTO tb_name (field1_name, ...) VALUES (field1_value1, ...); ``` 向表tb_name中插入一条记录,数据对应到指定的列。SQL语句中没有出现的列,数据库将自动填充为NULL。主键(时间戳)不能为NULL。 @@ -308,7 +331,8 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic ```mysql INSERT INTO tb_name VALUES (field1_value1, ...) (field1_value2, ...) ...; ``` - 向表tb_name中插入多条记录 + 向表tb_name中插入多条记录 + **注意**:在使用“插入多条记录”方式写入数据时,不能把第一列的时间戳取值都设为now,否则会导致语句中的多条记录使用相同的时间戳,于是就可能出现相互覆盖以致这些数据行无法全部被正确保存。 - **按指定的列插入多条记录** ```mysql @@ -330,29 +354,21 @@ TDengine缺省的时间戳是毫秒精度,但通过修改配置参数enableMic ``` 同时向表tb1_name和tb2_name中按列分别插入多条记录 - 注意: - 1) 如果时间戳为0,系统将自动使用服务器当前时间作为该记录的时间戳; - 2) 允许插入的最老记录的时间戳,是相对于当前服务器时间,减去配置的keep值(数据保留的天数),允许插入的最新记录的时间戳,是相对于当前服务器时间,加上配置的days值(数据文件存储数据的时间跨度,单位为天)。keep和days都是可以在创建数据库时指定的,缺省值分别是3650天和10天。 - -**历史记录写入**:可使用IMPORT或者INSERT命令,IMPORT的语法,功能与INSERT完全一样。 + 注意:允许插入的最老记录的时间戳,是相对于当前服务器时间,减去配置的keep值(数据保留的天数),允许插入的最新记录的时间戳,是相对于当前服务器时间,加上配置的days值(数据文件存储数据的时间跨度,单位为天)。keep和days都是可以在创建数据库时指定的,缺省值分别是3650天和10天。 -## 数据查询 +- **插入记录时自动建表** + ```mysql + INSERT INTO tb_name USING stb_name TAGS (tag_value1, ...) VALUES (field_value1, ...); + ``` + 如果用户在写数据时并不确定某个表是否存在,此时可以在写入数据时使用自动建表语法来创建不存在的表,若该表已存在则不会建立新表。自动建表时,要求必须以超级表为模板,并写明数据表的 tags 取值。 -### 查询语法: +- **插入记录时自动建表,并指定具体的 tags 列** + ```mysql + INSERT INTO tb_name USING stb_name (tag_name1, ...) TAGS (tag_value1, ...) VALUES (field_value1, ...); + ``` + 在自动建表时,可以只是指定部分 tags 列的取值,未被指定的 tags 列将取为空值。 -```mysql -SELECT select_expr [, select_expr ...] - FROM {tb_name_list} - [WHERE where_condition] - [INTERVAL (interval_val [, interval_offset])] - [FILL fill_val] - [SLIDING fill_val] - [GROUP BY col_list] - [ORDER BY col_list { DESC | ASC }] - [SLIMIT limit_val [, SOFFSET offset_val]] - [LIMIT limit_val [, OFFSET offset_val]] - [>> export_file] -``` +**历史记录写入**:可使用IMPORT或者INSERT命令,IMPORT的语法,功能与INSERT完全一样。 说明:针对 insert 类型的 SQL 语句,我们采用的流式解析策略,在发现后面的错误之前,前面正确的部分SQL仍会执行。下面的sql中,insert语句是无效的,但是d1001仍会被创建。 @@ -380,10 +396,30 @@ taos> SHOW TABLES; Query OK, 1 row(s) in set (0.001091s) ``` +## 数据查询 + +### 查询语法: + +```mysql +SELECT select_expr [, select_expr ...] + FROM {tb_name_list} + [WHERE where_condition] + [INTERVAL (interval_val [, interval_offset])] + [SLIDING sliding_val] + [FILL fill_val] + [GROUP BY col_list] + [ORDER BY col_list { DESC | ASC }] + [SLIMIT limit_val [, SOFFSET offset_val]] + [LIMIT limit_val [, OFFSET offset_val]] + [>> export_file]; +``` + #### SELECT子句 + 一个选择子句可以是联合查询(UNION)和另一个查询的子查询(SUBQUERY)。 ##### 通配符 + 通配符 * 可以用于代指全部列。对于普通表,结果中只有普通列。 ```mysql taos> SELECT * FROM d1001; @@ -466,6 +502,17 @@ Query OK, 2 row(s) in set (0.003112s) 注意:普通表的通配符 * 中并不包含 _标签列_。 +##### 获取标签列的去重取值 + +从 2.0.15 版本开始,支持在超级表查询标签列时,指定 distinct 关键字,这样将返回指定标签列的所有不重复取值。 +```mysql +SELECT DISTINCT tag_name FROM stb_name; +``` + +注意:目前 distinct 关键字只支持对超级表的标签列进行去重,而不能用于普通列。 + + + #### 结果集列名 ```SELECT```子句中,如果不指定返回结果集合的列名,结果集列名称默认使用```SELECT```子句中的表达式名称作为列名称。此外,用户可使用```AS```来重命名返回结果集合中列的名称。例如: @@ -481,6 +528,7 @@ Query OK, 3 row(s) in set (0.001191s) 但是针对```first(*)```、```last(*)```、```last_row(*)```不支持针对单列的重命名。 #### 隐式结果列 + ```Select_exprs```可以是表所属列的列名,也可以是基于列的函数表达式或计算式,数量的上限256个。当用户使用了```interval```或```group by tags```的子句以后,在最后返回结果中会强制返回时间戳列(第一列)和group by子句中的标签列。后续的版本中可以支持关闭group by子句中隐式列的输出,列输出完全由select子句控制。 #### 表(超级表)列表 @@ -495,6 +543,7 @@ SELECT * FROM d1001; ``` #### 特殊功能 + 部分特殊的查询功能可以不使用FROM子句执行。获取当前所在的数据库 database() ```mysql taos> SELECT DATABASE(); @@ -539,12 +588,14 @@ taos> SELECT SERVER_STATUS() AS status; 1 | Query OK, 1 row(s) in set (0.000081s) ``` + #### TAOS SQL中特殊关键词 > TBNAME: 在超级表查询中可视为一个特殊的标签,代表查询涉及的子表名
    \_c0: 表示表(超级表)的第一列 #### 小技巧 + 获取一个超级表所有的子表名及相关的标签信息: ```mysql SELECT TBNAME, location FROM meters; @@ -571,27 +622,30 @@ taos> SELECT COUNT(tbname) FROM meters WHERE groupId > 2; Query OK, 1 row(s) in set (0.001091s) ``` -- 可以使用* 返回所有列,或指定列名。可以对数字列进行四则运算,可以给输出的列取列名 -- where语句可以使用各种逻辑判断来过滤数字值,或使用通配符来过滤字符串 -- 输出结果缺省按首列时间戳升序排序,但可以指定按降序排序(_c0指首列时间戳)。使用ORDER BY对其他字段进行排序为非法操作。 -- 参数LIMIT控制输出条数,OFFSET指定从第几条开始输出。LIMIT/OFFSET对结果集的执行顺序在ORDER BY之后。 +- 可以使用 * 返回所有列,或指定列名。可以对数字列进行四则运算,可以给输出的列取列名 +- WHERE 语句可以使用各种逻辑判断来过滤数字值,或使用通配符来过滤字符串 +- 输出结果缺省按首列时间戳升序排序,但可以指定按降序排序( _c0 指首列时间戳)。使用 ORDER BY 对其他字段进行排序为非法操作。 +- 参数 LIMIT 控制输出条数,OFFSET 指定从第几条开始输出。LIMIT/OFFSET 对结果集的执行顺序在 ORDER BY 之后。 +- 参数 SLIMIT 控制由 GROUP BY 指令划分的每个分组中的输出条数。 - 通过”>>"输出结果可以导出到指定文件 ### 支持的条件过滤操作 -| Operation | Note | Applicable Data Types | -| --------- | ----------------------------- | ------------------------------------- | -| > | larger than | **`timestamp`** and all numeric types | -| < | smaller than | **`timestamp`** and all numeric types | -| >= | larger than or equal to | **`timestamp`** and all numeric types | -| <= | smaller than or equal to | **`timestamp`** and all numeric types | -| = | equal to | all types | -| <> | not equal to | all types | -| % | match with any char sequences | **`binary`** **`nchar`** | -| _ | match with a single char | **`binary`** **`nchar`** | +| Operation | Note | Applicable Data Types | +| ----------- | ----------------------------- | ------------------------------------- | +| > | larger than | **`timestamp`** and all numeric types | +| < | smaller than | **`timestamp`** and all numeric types | +| >= | larger than or equal to | **`timestamp`** and all numeric types | +| <= | smaller than or equal to | **`timestamp`** and all numeric types | +| = | equal to | all types | +| <> | not equal to | all types | +| between and | within a certain range | **`timestamp`** and all numeric types | +| % | match with any char sequences | **`binary`** **`nchar`** | +| _ | match with a single char | **`binary`** **`nchar`** | 1. 同时进行多个字段的范围过滤,需要使用关键词 AND 来连接不同的查询条件,暂不支持 OR 连接的不同列之间的查询过滤条件。 -2. 针对单一字段的过滤,如果是时间过滤条件,则一条语句中只支持设定一个;但针对其他的(普通)列或标签列,则可以使用``` OR``` 关键字进行组合条件的查询过滤。例如:((value > 20 and value < 30) OR (value < 12)) 。 +2. 针对单一字段的过滤,如果是时间过滤条件,则一条语句中只支持设定一个;但针对其他的(普通)列或标签列,则可以使用 `OR` 关键字进行组合条件的查询过滤。例如:((value > 20 AND value < 30) OR (value < 12)) 。 +3. 从 2.0.17 版本开始,条件过滤开始支持 BETWEEN AND 语法,例如 `WHERE col2 BETWEEN 1.5 AND 3.25` 表示查询条件为“1.5 ≤ col2 ≤ 3.25”。 ### SQL 示例 @@ -625,7 +679,7 @@ Query OK, 1 row(s) in set (0.001091s) SELECT COUNT(*) FROM tb1 WHERE ts >= NOW - 10m AND col2 > 3.14 >> /home/testoutpu.csv; ``` -## SQL 函数 +## SQL 函数 ### 聚合函数 @@ -697,13 +751,13 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ```mysql SELECT TWA(field_name) FROM tb_name WHERE clause; ``` - 功能说明:时间加权平均函数。统计表/超级表中某列在一段时间内的时间加权平均。 + 功能说明:时间加权平均函数。统计表中某列在一段时间内的时间加权平均。 返回结果数据类型:双精度浮点数Double。 应用字段:不能应用在timestamp、binary、nchar、bool类型字段。 - 适用于:表、超级表。 + 适用于:表。 - **SUM** ```mysql @@ -742,7 +796,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 应用字段:不能应用在timestamp、binary、nchar、bool类型字段。 - 适用于:表。 + 适用于:表。(从 2.0.15.1 版本开始,本函数也支持超级表) 示例: ```mysql @@ -1104,38 +1158,41 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 3 row(s) in set (0.001046s) ``` -## 时间维度聚合 +## 时间维度聚合 + TDengine支持按时间段进行聚合,可以将表中数据按照时间段进行切割后聚合生成结果,比如温度传感器每秒采集一次数据,但需查询每隔10分钟的温度平均值。这个聚合适合于降维(down sample)操作, 语法如下: ```mysql SELECT function_list FROM tb_name [WHERE where_condition] INTERVAL (interval [, offset]) + [SLIDING sliding] [FILL ({NONE | VALUE | PREV | NULL | LINEAR})] SELECT function_list FROM stb_name [WHERE where_condition] INTERVAL (interval [, offset]) + [SLIDING sliding] [FILL ({ VALUE | PREV | NULL | LINEAR})] [GROUP BY tags] ``` - 聚合时间段的长度由关键词INTERVAL指定,最短时间间隔10毫秒(10a),并且支持偏移(偏移必须小于间隔)。聚合查询中,能够同时执行的聚合和选择函数仅限于单个输出的函数:count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last,不能使用具有多行输出结果的函数(例如:top、bottom、diff以及四则运算)。 - WHERE语句可以指定查询的起止时间和其他过滤条件 +- SLIDING语句用于指定聚合时间段的前向增量 - FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种: - 1. 不进行填充:NONE(默认填充模式)。 - - 2. VALUE填充:固定值填充,此时需要指定填充的数值。例如:fill(value, 1.23)。 - - 3. NULL填充:使用NULL填充数据。例如:fill(null)。 - - 4. PREV填充:使用前一个非NULL值填充数据。例如:fill(prev)。 + * 不进行填充:NONE(默认填充模式)。 + * VALUE填充:固定值填充,此时需要指定填充的数值。例如:fill(value, 1.23)。 + * NULL填充:使用NULL填充数据。例如:fill(null)。 + * PREV填充:使用前一个非NULL值填充数据。例如:fill(prev)。 说明: 1. 使用FILL语句的时候可能生成大量的填充输出,务必指定查询的时间区间。针对每次查询,系统可返回不超过1千万条具有插值的结果。 2. 在时间维度聚合中,返回的结果中时间序列严格单调递增。 3. 如果查询对象是超级表,则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用group by语句,则返回的结果按照时间序列严格单调递增;如果查询中使用了group by语句分组,则返回结果中每个group内不按照时间序列严格单调递增。 +时间聚合也常被用于连续查询场景,可以参考文档 [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation/advanced-features#continuous-query)。 + **示例:** 智能电表的建表语句如下: ```mysql @@ -1152,12 +1209,13 @@ SELECT AVG(current), MAX(current), LEASTSQUARES(current, start_val, step_val), P FILL(PREV); ``` -## TAOS SQL 边界限制 -- 数据库名最大长度为32 -- 表名最大长度为192,每行数据最大长度16k个字符 -- 列名最大长度为64,最多允许1024列,最少需要2列,第一列必须是时间戳 -- 标签最多允许128个,可以0个,标签总长度不超过16k个字符 -- SQL语句最大长度65480个字符,但可通过系统配置参数maxSQLLength修改,最长可配置为1M +## TAOS SQL 边界限制 + +- 数据库名最大长度为 32 +- 表名最大长度为 192,每行数据最大长度 16k 个字符(注意:数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置) +- 列名最大长度为 64,最多允许 1024 列,最少需要 2 列,第一列必须是时间戳 +- 标签最多允许 128 个,可以 1 个,标签总长度不超过 16k 个字符 +- SQL 语句最大长度 65480 个字符,但可通过系统配置参数 maxSQLLength 修改,最长可配置为 1M - 库的数目,超级表的数目、表的数目,系统不做限制,仅受系统资源限制 ## TAOS SQL其他约定 @@ -1172,4 +1230,5 @@ TAOS SQL支持表之间按主键时间戳来join两张表的列,暂不支持 **is not null与不为空的表达式适用范围** -is not null支持所有类型的列。不为空的表达式为 <>"",仅对非数值类型的列适用。 \ No newline at end of file +is not null支持所有类型的列。不为空的表达式为 <>"",仅对非数值类型的列适用。 + diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/cn/13.faq/docs.md similarity index 56% rename from documentation20/webdocs/markdowndocs/faq-ch.md rename to documentation20/cn/13.faq/docs.md index a085e6159a37aec1699e3e5100ddeec6d71f4fb6..c01247d345906bde021d88841d34f667c8991fa9 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/cn/13.faq/docs.md @@ -1,23 +1,40 @@ # 常见问题 +## 0. 怎么报告问题? + +如果 FAQ 中的信息不能够帮到您,需要 TDengine 技术团队的技术支持与协助,请将以下两个目录中内容打包: +1. /var/log/taos (如果没有修改过默认路径) +2. /etc/taos + +附上必要的问题描述,包括使用的 TDengine 版本信息、平台环境信息、发生该问题的执行操作、出现问题的表征及大概的时间,在 [GitHub](https://github.com/taosdata/TDengine) 提交Issue。 + +为了保证有足够的debug信息,如果问题能够重复,请修改/etc/taos/taos.cfg文件,最后面添加一行“debugFlag 135"(不带引号本身),然后重启taosd, 重复问题,然后再递交。也可以通过如下SQL语句,临时设置taosd的日志级别。 +``` + alter dnode debugFlag 135; +``` +但系统正常运行时,请一定将debugFlag设置为131,否则会产生大量的日志信息,降低系统效率。 + ## 1. TDengine2.0之前的版本升级到2.0及以上的版本应该注意什么?☆☆☆ -2.0版本在之前版本的基础上,进行了完全的重构,配置文件和数据文件是不兼容的。在升级之前务必进行如下操作: +2.0版在之前版本的基础上,进行了完全的重构,配置文件和数据文件是不兼容的。在升级之前务必进行如下操作: -1. 删除配置文件,执行 sudo rm -rf /etc/taos/taos.cfg -2. 删除日志文件,执行 sudo rm -rf /var/log/taos/ -3. 确保数据已经不再需要的前提下,删除数据文件,执行 sudo rm -rf /var/lib/taos/ -4. 安装最新稳定版本的TDengine -5. 如果数据需要迁移数据或者数据文件损坏,请联系涛思数据官方技术支持团队,进行协助解决 +1. 删除配置文件,执行 `sudo rm -rf /etc/taos/taos.cfg` +2. 删除日志文件,执行 `sudo rm -rf /var/log/taos/` +3. 确保数据已经不再需要的前提下,删除数据文件,执行 `sudo rm -rf /var/lib/taos/` +4. 安装最新稳定版本的 TDengine +5. 如果需要迁移数据或者数据文件损坏,请联系涛思数据官方技术支持团队,进行协助解决 ## 2. Windows平台下JDBCDriver找不到动态链接库,怎么办? -请看为此问题撰写的技术博客 + +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/jdbcdriver找不到动态链接库/) ## 3. 创建数据表时提示more dnodes are needed -请看为此问题撰写的技术博客 + +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/03/创建数据表时提示more-dnodes-are-needed/) ## 4. 如何让TDengine crash时生成core文件? -请看为此问题撰写的技术博客 + +请看为此问题撰写的[技术博客](https://www.taosdata.com/blog/2019/12/06/tdengine-crash时生成core文件的方法/) ## 5. 遇到错误"Unable to establish connection", 我怎么办? @@ -32,7 +49,7 @@ 3. 在服务器,执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行,启动*taosd* -4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)),FQDN配置参考:一篇文章说清楚TDengine的FQDN。 +4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得)),FQDN配置参考:[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。 5. ping服务器FQDN,如果没有反应,请检查你的网络,DNS设置,或客户端所在计算机的系统hosts文件 @@ -51,12 +68,12 @@ * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问 -10. 也可以使用taos程序内嵌的网络连通检测功能,来验证服务器和客户端之间指定的端口连接是否通畅(包括TCP和UDP):TDengine 内嵌网络检测工具使用指南。 +10. 也可以使用taos程序内嵌的网络连通检测功能,来验证服务器和客户端之间指定的端口连接是否通畅(包括TCP和UDP):[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。 ## 6. 遇到错误“Unexpected generic error in RPC”或者"TDengine Error: Unable to resolve FQDN", 我怎么办? 产生这个错误,是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用,请做如下检查: -1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考:一篇文章说清楚TDengine的FQDN。 +1. 请检查连接的服务器的FQDN是否正确,FQDN配置参考:[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。 2. 如果网络配置有DNS server, 请检查是否正常工作 3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件,查看该FQDN是否配置,并是否有正确的IP地址。 4. 如果网络配置OK,从客户端所在机器,你需要能Ping该连接的FQDN,否则客户端是无法连接服务器的 @@ -71,17 +88,21 @@ TDengine还没有一组专用的validation queries。然而建议你使用系统 ## 9. 我可以删除或更新一条记录吗? -不能。因为TDengine是为联网设备采集的数据设计的,不容许修改。但TDengine提供数据保留策略,只要数据记录超过保留时长,就会被自动删除。 +TDengine 目前尚不支持删除功能,未来根据用户需求可能会支持。 + +从 2.0.8.0 开始,TDengine 支持更新已经写入数据的功能。使用更新功能需要在创建数据库时使用 UPDATE 1 参数,之后可以使用 INSERT INTO 命令更新已经写入的相同时间戳数据。UPDATE 参数不支持 ALTER DATABASE 命令修改。没有使用 UPDATE 1 参数创建的数据库,写入相同时间戳的数据不会修改之前的数据,也不会报错。 + +另需注意,在 UPDATE 设置为 0 时,后发送的相同时间戳的数据会被直接丢弃,但并不会报错,而且仍然会被计入 affected rows (所以不能利用 INSERT 指令的返回信息进行时间戳查重)。这样设计的主要原因是,TDengine 把写入的数据看做一个数据流,无论时间戳是否出现冲突,TDengine 都认为产生数据的原始设备真实地产生了这样的数据。UPDATE 参数只是控制这样的流数据在进行持久化时要怎样处理——UPDATE 为 0 时,表示先写入的数据覆盖后写入的数据;而 UPDATE 为 1 时,表示后写入的数据覆盖先写入的数据。这种覆盖关系如何选择,取决于对数据的后续使用和统计中,希望以先还是后生成的数据为准。 ## 10. 我怎么创建超过1024列的表? 使用2.0及其以上版本,默认支持1024列;2.0之前的版本,TDengine最大允许创建250列的表。但是如果确实超过限值,建议按照数据特性,逻辑地将这个宽表分解成几个小表。 -## 10. 最有效的写入数据的方法是什么? +## 11. 最有效的写入数据的方法是什么? 批量插入。每条写入语句可以一张表同时插入多条记录,也可以同时插入多张表的多条记录。 -## 11. 最有效的写入数据的方法是什么?windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决? +## 12. 最有效的写入数据的方法是什么?windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决? Windows下插入nchar类的数据中如果有中文,请先确认系统的地区设置成了中国(在Control Panel里可以设置),这时cmd中的`taos`客户端应该已经可以正常工作了;如果是在IDE里开发Java应用,比如Eclipse, Intellij,请确认IDE里的文件编码为GBK(这是Java默认的编码类型),然后在生成Connection时,初始化客户端的配置,具体语句如下: ```JAVA @@ -90,10 +111,9 @@ Properties properties = new Properties(); properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); Connection = DriverManager.getConnection(url, properties); ``` -## 12.TDengine GO windows驱动的如何编译? -请看为此问题撰写的技术博客 ## 13.JDBC报错: the excuted SQL is not a DML or a DDL? + 请更新至最新的JDBC驱动 ```JAVA @@ -102,6 +122,7 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` + ## 14. taos connect failed, reason: invalid timestamp 常见原因是服务器和客户端时间没有校准,可以通过和时间服务器同步的方式(Linux 下使用 ntpdate 命令,Windows 在系统时间设置中选择自动同步)校准。 @@ -118,16 +139,30 @@ TDengine是根据hostname唯一标志一台机器的,在数据文件从机器A - 2.0.7.0 及以后的版本,到/var/lib/taos/dnode下,修复dnodeEps.json的dnodeId对应的FQDN,重启。确保机器内所有机器的此文件是完全相同的。 - 1.x 和 2.x 版本的存储结构不兼容,需要使用迁移工具或者自己开发应用导出导入数据。 -## 17. 怎么报告问题? - -如果 FAQ 中的信息不能够帮到您,需要 TDengine 技术团队的技术支持与协助,请将以下两个目录中内容打包: -1. /var/log/taos -2. /etc/taos +## 17. 如何在命令行程序 taos 中临时调整日志级别 -附上必要的问题描述,以及发生该问题的执行操作,出现问题的表征及大概的时间,在 GitHub提交Issue。 +为了调试方便,从 2.0.16 版本开始,命令行程序 taos 新增了与日志记录相关的两条指令: -为了保证有足够的debug信息,如果问题能够重复,请修改/etc/taos/taos.cfg文件,最后面添加一行“debugFlag 135"(不带引号本身),然后重启taosd, 重复问题,然后再递交。也可以通过如下SQL语句,临时设置taosd的日志级别。 +```mysql +ALTER LOCAL flag_name flag_value; ``` - alter dnode debugFlag 135; + +其含义是,在当前的命令行程序下,修改一个特定模块的日志记录级别(只对当前命令行程序有效,如果 taos 命令行程序重启,则需要重新设置): +- flag_name 的取值可以是:debugFlag,cDebugFlag,tmrDebugFlag,uDebugFlag,rpcDebugFlag +- flag_value 的取值可以是:131(输出错误和警告日志),135( 输出错误、警告和调试日志),143( 输出错误、警告、调试和跟踪日志) + +```mysql +ALTER LOCAL RESETLOG; ``` -但系统正常运行时,请一定将debugFlag设置为131,否则会产生大量的日志信息,降低系统效率。 + +其含义是,清空本机所有由客户端生成的日志文件。 + +## 18. 时间戳的时区信息是怎样处理的? + +TDengine 中时间戳的时区总是由客户端进行处理,而与服务端无关。具体来说,客户端会对 SQL 语句中的时间戳进行时区转换,转为 UTC 时区(即 Unix 时间戳——Unix Timestamp)再交由服务端进行写入和查询;在读取数据时,服务端也是采用 UTC 时区提供原始数据,客户端收到后再根据本地设置,把时间戳转换为本地系统所要求的时区进行显示。 + +客户端在处理时间戳字符串时,会采取如下逻辑: +1. 在未做特殊设置的情况下,客户端默认使用所在操作系统的时区设置。 +2. 如果在 taos.cfg 中设置了 timezone 参数,则客户端会以这个配置文件中的设置为准。 +3. 如果在 C/C++/Java/Python 等各种编程语言的 Connector Driver 中,在建立数据库连接时显式指定了 timezone,那么会以这个指定的时区设置为准。例如 Java Connector 的 JDBC URL 中就有 timezone 参数。 +4. 在书写 SQL 语句时,也可以直接使用 Unix 时间戳(例如 `1554984068000`)或带有时区的时间戳字符串,也即以 RFC 3339 格式(例如 `2013-04-12T15:52:01.123+08:00`)或 ISO-8601 格式(例如 `2013-04-12T15:52:01.123+0800`)来书写时间戳,此时这些时间戳的取值将不再受其他时区设置的影响。 diff --git a/documentation20/webdocs/assets/dnode.png b/documentation20/cn/images/architecture/dnode.png similarity index 100% rename from documentation20/webdocs/assets/dnode.png rename to documentation20/cn/images/architecture/dnode.png diff --git a/documentation20/webdocs/assets/message.png b/documentation20/cn/images/architecture/message.png similarity index 100% rename from documentation20/webdocs/assets/message.png rename to documentation20/cn/images/architecture/message.png diff --git a/documentation20/webdocs/assets/modules.png b/documentation20/cn/images/architecture/modules.png similarity index 100% rename from documentation20/webdocs/assets/modules.png rename to documentation20/cn/images/architecture/modules.png diff --git a/documentation20/cn/images/architecture/multi_tables.png b/documentation20/cn/images/architecture/multi_tables.png new file mode 100644 index 0000000000000000000000000000000000000000..0cefaab6a9a4cdd671c671f7c6186dea41415ff0 Binary files /dev/null and b/documentation20/cn/images/architecture/multi_tables.png differ diff --git a/documentation20/cn/images/architecture/replica-forward.png b/documentation20/cn/images/architecture/replica-forward.png new file mode 100644 index 0000000000000000000000000000000000000000..bf616e030b130603eceb5dccfd30b4a1dfa68ea5 Binary files /dev/null and b/documentation20/cn/images/architecture/replica-forward.png differ diff --git a/documentation20/webdocs/assets/replica-master.png b/documentation20/cn/images/architecture/replica-master.png similarity index 100% rename from documentation20/webdocs/assets/replica-master.png rename to documentation20/cn/images/architecture/replica-master.png diff --git a/documentation20/cn/images/architecture/replica-restore.png b/documentation20/cn/images/architecture/replica-restore.png new file mode 100644 index 0000000000000000000000000000000000000000..1558e5ed0108d23efdc6b5d9ea0e44a1dff45d28 Binary files /dev/null and b/documentation20/cn/images/architecture/replica-restore.png differ diff --git a/documentation20/cn/images/architecture/structure.png b/documentation20/cn/images/architecture/structure.png new file mode 100644 index 0000000000000000000000000000000000000000..4fc8f47ab0a30d95b85ba1d85105726ed981e56e Binary files /dev/null and b/documentation20/cn/images/architecture/structure.png differ diff --git a/documentation20/webdocs/assets/vnode.png b/documentation20/cn/images/architecture/vnode.png similarity index 100% rename from documentation20/webdocs/assets/vnode.png rename to documentation20/cn/images/architecture/vnode.png diff --git a/documentation20/webdocs/assets/write_master.png b/documentation20/cn/images/architecture/write_master.png similarity index 100% rename from documentation20/webdocs/assets/write_master.png rename to documentation20/cn/images/architecture/write_master.png diff --git a/documentation20/webdocs/assets/write_slave.png b/documentation20/cn/images/architecture/write_slave.png similarity index 100% rename from documentation20/webdocs/assets/write_slave.png rename to documentation20/cn/images/architecture/write_slave.png diff --git a/documentation20/cn/images/connections/add_datasource1.jpg b/documentation20/cn/images/connections/add_datasource1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..1f0f5110f312c57f3ec1788bbc02f04fac6ac142 Binary files /dev/null and b/documentation20/cn/images/connections/add_datasource1.jpg differ diff --git a/documentation20/cn/images/connections/add_datasource2.jpg b/documentation20/cn/images/connections/add_datasource2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fa7a83e00e96fae649910dff4edf5f5bdadd7850 Binary files /dev/null and b/documentation20/cn/images/connections/add_datasource2.jpg differ diff --git a/documentation20/cn/images/connections/add_datasource3.jpg b/documentation20/cn/images/connections/add_datasource3.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fc850ad08ff1174de972906842e0d5ee64e6e5cb Binary files /dev/null and b/documentation20/cn/images/connections/add_datasource3.jpg differ diff --git a/documentation20/cn/images/connections/add_datasource4.jpg b/documentation20/cn/images/connections/add_datasource4.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3ba73e50d455111f8621f4165746078554c2d790 Binary files /dev/null and b/documentation20/cn/images/connections/add_datasource4.jpg differ diff --git a/documentation20/cn/images/connections/create_dashboard1.jpg b/documentation20/cn/images/connections/create_dashboard1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..3b83c3a1714e9e7540e0b06239ef7c1c4f63fe2c Binary files /dev/null and b/documentation20/cn/images/connections/create_dashboard1.jpg differ diff --git a/documentation20/cn/images/connections/create_dashboard2.jpg b/documentation20/cn/images/connections/create_dashboard2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fe5d768ac55254251e0290bf257178f5ff28f5a5 Binary files /dev/null and b/documentation20/cn/images/connections/create_dashboard2.jpg differ diff --git a/documentation20/cn/images/connections/import_dashboard1.jpg b/documentation20/cn/images/connections/import_dashboard1.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9d2ce7ed65eb0c2c729de50283b30491793493dc Binary files /dev/null and b/documentation20/cn/images/connections/import_dashboard1.jpg differ diff --git a/documentation20/cn/images/connections/import_dashboard2.jpg b/documentation20/cn/images/connections/import_dashboard2.jpg new file mode 100644 index 0000000000000000000000000000000000000000..94b09f0ee39552bb84f7ba1f65815ce2c9548b2d Binary files /dev/null and b/documentation20/cn/images/connections/import_dashboard2.jpg differ diff --git a/documentation20/webdocs/assets/connector.png b/documentation20/cn/images/connector.png similarity index 100% rename from documentation20/webdocs/assets/connector.png rename to documentation20/cn/images/connector.png diff --git a/documentation20/webdocs/assets/EcoSystem.png b/documentation20/cn/images/eco_system.png similarity index 100% rename from documentation20/webdocs/assets/EcoSystem.png rename to documentation20/cn/images/eco_system.png diff --git a/documentation20/cn/images/tdengine-jdbc-connector.png b/documentation20/cn/images/tdengine-jdbc-connector.png new file mode 100644 index 0000000000000000000000000000000000000000..fdf1dd3fcc5ee222c4a8753efa2c95c5257314bf Binary files /dev/null and b/documentation20/cn/images/tdengine-jdbc-connector.png differ diff --git a/documentation20/webdocs/assets/replica-forward.jpg b/documentation20/webdocs/assets/replica-forward.jpg deleted file mode 100644 index 00b8c357b9ca2f067127c7dfae37b386b268f8dd..0000000000000000000000000000000000000000 Binary files a/documentation20/webdocs/assets/replica-forward.jpg and /dev/null differ diff --git a/documentation20/webdocs/assets/replica-restore.jpg b/documentation20/webdocs/assets/replica-restore.jpg deleted file mode 100644 index 6da1bed4cc90b8a119b2fddf5f66ef3ab1b0a2fd..0000000000000000000000000000000000000000 Binary files a/documentation20/webdocs/assets/replica-restore.jpg and /dev/null differ diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md deleted file mode 100644 index 20f9bcb7307c68f1330519d8f6ed249419bf46db..0000000000000000000000000000000000000000 --- a/documentation20/webdocs/markdowndocs/Documentation-ch.md +++ /dev/null @@ -1,145 +0,0 @@ -# TDengine文档 - -TDengine是一个高效的存储、查询、分析时序大数据的平台,专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它,但建议您在使用前仔细阅读一遍下面的文档,特别是[数据模型](https://www.taosdata.com/cn/documentation20/data-model-and-architecture)与[数据建模](https://www.taosdata.com/cn/documentation20/model)一节。除本文档之外,欢迎[下载产品白皮书](https://www.taosdata.com/downloads/TDengine White Paper.pdf)。如需查阅TDengine 1.6 文档,请点击[这里](https://www.taosdata.com/cn/documentation16/)访问。 - -## TDengine介绍 - -- TDengine 简介及特色 -- TDengine 适用场景 - -## [立即开始](https://www.taosdata.com/cn/getting-started) - -- [快捷安装](https://www.taosdata.com/cn/documentation20/getting-started/#快捷安装):可通过源码、安装包或docker安装,三秒钟搞定 -- [轻松启动](https://www.taosdata.com/cn/documentation20/getting-started/#轻松启动):使用systemctl 启停TDengine -- [命令行程序TAOS](https://www.taosdata.com/cn/documentation20/getting-started/#TDengine命令行程序):访问TDengine的简便方式 -- [极速体验](https://www.taosdata.com/cn/documentation20/getting-started/#TDengine-极速体验):运行示例程序,快速体验高效的数据插入、查询 - -## [数据模型和整体架构](https://www.taosdata.com/cn/documentation20/architecture) - -- [数据模型](https://www.taosdata.com/cn/documentation20/architecture/#数据模型):关系型数据库模型,但要求每个采集点单独建表 -- [集群与基本逻辑单元](https://www.taosdata.com/cn/documentation20/architecture/#集群与基本逻辑单元):吸取NoSQL优点,支持水平扩展,支持高可靠 -- [存储模型与数据分区、分片](https://www.taosdata.com/cn/documentation20/architecture/#存储模型与数据分区、分片):标签数据与时序数据完全分离,按vnode和时间两个维度对数据切分 -- [数据写入与复制流程](https://www.taosdata.com/cn/documentation20/architecture/#数据写入与复制流程):先写入WAL、之后写入缓存,再给应用确认,支持多副本 -- [缓存与持久化](https://www.taosdata.com/cn/documentation20/architecture/#缓存与持久化):最新数据缓存在内存中,但落盘时采用列式存储、超高压缩比 -- [数据查询](https://www.taosdata.com/cn/documentation20/architecture/#数据查询):支持各种函数、时间轴聚合、插值、多表聚合 - -## [数据建模](https://www.taosdata.com/cn/documentation20/model) - -- [创建库](https://www.taosdata.com/cn/documentation20/model/#创建库):为具有相似数据特征的数据采集点创建一个库 -- [创建超级表](https://www.taosdata.com/cn/documentation20/model/#创建超级表):为同一类型的数据采集点创建一个超级表 -- [创建表](https://www.taosdata.com/cn/documentation20/model/#创建表):使用超级表做模板,为每一个具体的数据采集点单独建表 - -## [高效写入数据](https://www.taosdata.com/cn/documentation20/insert) - -- [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入):使用SQL insert命令向一张或多张表写入单条或多条记录 -- [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入):配置Telegraf, 不用任何代码,将采集数据直接写入 -- [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入):配置Prometheus, 不用任何代码,将数据直接写入 -- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入):配置EMQ X,不用任何代码,就可将 MQTT 数据直接写入 -- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker直接写入):通过 HiveMQ Extension,不用任何代码,就可将 MQTT 数据直接写入 - -## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries) - -- [主要查询功能](https://www.taosdata.com/cn/documentation20/queries/#主要查询功能):支持各种标准函数,设置过滤条件,时间段查询 -- [多表聚合查询](https://www.taosdata.com/cn/documentation20/queries/#多表聚合查询):使用超级表,设置标签过滤条件,进行高效聚合查询 -- [降采样查询值](https://www.taosdata.com/cn/documentation20/queries/#降采样查询、插值):按时间段分段聚合,支持插值 - -## [高级功能](https://www.taosdata.com/cn/documentation20/advanced-features) - -- [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation20/advanced-features/#连续查询(Continuous-Query)):基于滑动窗口,定时自动的对数据流进行查询计算 -- [数据订阅(Publisher/Subscriber)](https://www.taosdata.com/cn/documentation20/advanced-features/#数据订阅(Publisher/Subscriber)):象典型的消息队列,应用可订阅接收到的最新数据 -- [缓存(Cache)](https://www.taosdata.com/cn/documentation20/advanced-features/#缓存(Cache)):每个设备最新的数据都会缓存在内存中,可快速获取 -- [报警监测](https://www.taosdata.com/cn/documentation20/advanced-features/#报警监测(Alert)):根据配置规则,自动监测超限行为数据,并主动推送 - -## [连接器](https://www.taosdata.com/cn/documentation20/connector) - -- [C/C++ Connector](https://www.taosdata.com/cn/documentation20/connector/#C/C++-Connector):通过libtaos客户端的库,连接TDengine服务器的主要方法 -- [Java Connector(JDBC)](https://www.taosdata.com/cn/documentation20/connector-java):通过标准的JDBC API,给Java应用提供到TDengine的连接 -- [Python Connector](https://www.taosdata.com/cn/documentation20/connector/#Python-Connector):给Python应用提供一个连接TDengine服务器的驱动 -- [RESTful Connector](https://www.taosdata.com/cn/documentation20/connector/#RESTful-Connector):提供一最简单的连接TDengine服务器的方式 -- [Go Connector](https://www.taosdata.com/cn/documentation20/connector/#Go-Connector):给Go应用提供一个连接TDengine服务器的驱动 -- [Node.js Connector](https://www.taosdata.com/cn/documentation20/connector/#Node.js-Connector):给node应用提供一个链接TDengine服务器的驱动 - -## [与其他工具的连接](https://www.taosdata.com/cn/documentation20/connections-with-other-tools) - -- [Grafana](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#Grafana):获取并可视化保存在TDengine的数据 -- [Matlab](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#Matlab):通过配置Matlab的JDBC数据源访问保存在TDengine的数据 -- [R](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#R):通过配置R的JDBC数据源访问保存在TDengine的数据 - -## [TDengine集群的安装、管理](https://www.taosdata.com/cn/documentation20/cluster) - -- [安装](https://www.taosdata.com/cn/documentation20/cluster/#创建第一个节点):与单节点的安装一样,但要设好配置文件里的参数first -- [节点管理](https://www.taosdata.com/cn/documentation20/cluster/#节点管理):增加、删除、查看集群的节点 -- [mnode的管理](https://www.taosdata.com/cn/documentation20/cluster/#Mnode的高可用):系统自动创建、无需任何人工干预 -- [负载均衡](https://www.taosdata.com/cn/documentation20/cluster/#负载均衡):一旦节点个数或负载有变化,自动进行 -- [节点离线处理](https://www.taosdata.com/cn/documentation20/cluster/#节点离线处理):节点离线超过一定时长,将从集群中剔除 -- [Arbitrator](https://www.taosdata.com/cn/documentation20/cluster/#Arbitrator的使用):对于偶数个副本的情形,使用它可以防止split brain - -## [TDengine的运营和维护](https://www.taosdata.com/cn/documentation20/administrator) - -- [容量规划](https://www.taosdata.com/cn/documentation20/administrator/#容量规划):根据场景,估算硬件资源 -- [容错和灾备](https://www.taosdata.com/cn/documentation20/administrator/#容错和灾备):设置正确的WAL和数据副本数 -- [系统配置](https://www.taosdata.com/cn/documentation20/administrator/#服务端配置):端口,缓存大小,文件块大小和其他系统配置 -- [用户管理](https://www.taosdata.com/cn/documentation20/administrator/#用户管理):添加、删除TDengine用户,修改用户密码 -- [数据导入](https://www.taosdata.com/cn/documentation20/administrator/#数据导入):可按脚本文件导入,也可按数据文件导入 -- [数据导出](https://www.taosdata.com/cn/documentation20/administrator/#数据导出):从shell按表导出,也可用taosdump工具做各种导出 -- [系统监控](https://www.taosdata.com/cn/documentation20/administrator/#系统监控):检查系统现有的连接、查询、流式计算,日志和事件等 -- [文件目录结构](https://www.taosdata.com/cn/documentation20/administrator/#文件目录结构):TDengine数据文件、配置文件等所在目录 -- [参数限制和保留关键字](https://www.taosdata.com/cn/documentation20/administrator/#参数限制和保留关键字):TDengine的参数限制和保留关键字列表 - -## [TAOS SQL](https://www.taosdata.com/cn/documentation20/taos-sql) - -- [支持的数据类型](https://www.taosdata.com/cn/documentation20/taos-sql/#支持的数据类型):支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 -- [数据库管理](https://www.taosdata.com/cn/documentation20/taos-sql/#数据库管理):添加、删除、查看数据库 -- [表管理](https://www.taosdata.com/cn/documentation20/taos-sql/#表管理):添加、删除、查看、修改表 -- [超级表管理](https://www.taosdata.com/cn/documentation20/taos-sql/#超级表STable管理):添加、删除、查看、修改超级表 -- [标签管理](https://www.taosdata.com/cn/documentation20/taos-sql/#超级表-STable-中-TAG-管理):增加、删除、修改标签 -- [数据写入](https://www.taosdata.com/cn/documentation20/taos-sql/#数据写入):支持单表单条、多条、多表多条写入,支持历史数据写入 -- [数据查询](https://www.taosdata.com/cn/documentation20/taos-sql/#数据查询):支持时间段、值过滤、排序、查询结果手动分页等 -- [SQL函数](https://www.taosdata.com/cn/documentation20/taos-sql/#SQL函数):支持各种聚合函数、选择函数、计算函数,如avg, min, diff等 -- [时间维度聚合](https://www.taosdata.com/cn/documentation20/taos-sql/#时间维度聚合):将表中数据按照时间段进行切割后聚合,降维处理 -- [边界线制](https://www.taosdata.com/cn/documentation20/taos-sql/#TAOS-SQL-边界限制):TAOS SQL的边界限制 -- [错误码](https://www.taosdata.com/cn/documentation20/Taos-Error-Code):TDengine 2.0 错误码以及对应的十进制码 - -## TDengine的技术设计 - -- 系统模块:taosd的功能和模块划分 -- 数据复制:支持实时同步、异步复制,保证系统的High Availibility -- [技术博客](https://www.taosdata.com/cn/blog/?categories=3):更多的技术分析和架构设计文章 - -## 常用工具 - -- [TDengine样例导入工具](https://www.taosdata.com/blog/2020/01/18/1166.html) -- [TDengine性能对比测试工具](https://www.taosdata.com/blog/2020/01/18/1166.html) - -## TDengine与其他数据库的对比测试 - -- [用InfluxDB开源的性能测试工具对比InfluxDB和TDengine](https://www.taosdata.com/blog/2020/01/13/1105.html) -- [TDengine与OpenTSDB对比测试](https://www.taosdata.com/blog/2019/08/21/621.html) -- [TDengine与Cassandra对比测试](https://www.taosdata.com/blog/2019/08/14/573.html) -- [TDengine与InfluxDB对比测试](https://www.taosdata.com/blog/2019/07/19/419.html) -- [TDengine与InfluxDB、OpenTSDB、Cassandra、MySQL、ClickHouse等数据库的对比测试报告](https://www.taosdata.com/downloads/TDengine_Testing_Report_cn.pdf) - -##物联网大数据 - -- [物联网、工业互联网大数据的特点](https://www.taosdata.com/blog/2019/07/09/105.html) -- [物联网大数据平台应具备的功能和特点](https://www.taosdata.com/blog/2019/07/29/542.html) -- [通用大数据架构为什么不适合处理物联网数据?](https://www.taosdata.com/blog/2019/07/09/107.html) -- [物联网、车联网、工业互联网大数据平台,为什么推荐使用TDengine?](https://www.taosdata.com/blog/2019/07/09/109.html) - -## [培训和FAQ](https://www.taosdata.com/cn/faq) - - - diff --git a/documentation20/webdocs/markdowndocs/cluster.md b/documentation20/webdocs/markdowndocs/cluster.md deleted file mode 100644 index 08206a85a8954689efb824fed6ed7e9572000a0b..0000000000000000000000000000000000000000 --- a/documentation20/webdocs/markdowndocs/cluster.md +++ /dev/null @@ -1,146 +0,0 @@ -#集群安装、管理 - -多个taosd的运行实例可以组成一个集群,以保证TDengine的高可靠运行,并提供水平扩展能力。要了解TDengine 2.0的集群管理,需要对集群的基本概念有所了解,请看TDengine 2.0整体架构一章。 - -集群的每个节点是由End Point来唯一标识的,End Point是由FQDN(Fully Qualified Domain Name)外加Port组成,比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname,可通过Linux命令“hostname"获取。端口是这个节点对外服务的端口号,缺省是6030,但可以通过taos.cfg里配置参数serverPort进行修改。 - -TDengine的集群管理极其简单,除添加和删除节点需要人工干预之外,其他全部是自动完成,最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 - -##安装、创建第一个节点 - -集群是由一个一个dnode组成的,是从一个dnode的创建开始的。创建第一个节点很简单,就按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法进行安装、启动即可。 - -启动后,请执行taos, 启动taos shell,从shell里执行命令"show dnodes;",如下所示: - ``` -Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 -Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. - -taos> show dnodes; - id | end_point | vnodes | cores | status | role | create_time | -===================================================================================== - 1 | h1.taos.com:6030 | 0 | 2 | ready | any | 2020-07-31 03:49:29.202 | -Query OK, 1 row(s) in set (0.006385s) - -taos> - ``` -上述命令里,可以看到这个刚启动的这个节点的End Point是:h1.taos.com:6030 - -## 安装、创建后续节点 - -将新的节点添加到现有集群,具体有以下几步: - -1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法进行安装,但不要启动taosd - -2. 如果是使用涛思数据的官方安装包进行安装,在安装结束时,会询问集群的End Port, 输入第一个节点的End Point即可。如果是源码安装,请编辑配置文件taos.cfg(缺省是在/etc/taos/目录),增加一行: - - ``` - firstEp h1.taos.com:6030 - ``` - - 请注意将示例的“h1.taos.com:6030" 替换为你自己第一个节点的End Point - -3. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法启动taosd - -4. 在Linux shell里执行命令"hostname"找出本机的FQDN, 假设为h2.taos.com。如果无法找到,可以查看taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录),fqdn以及port都会打印出来。 - -5. 在第一个节点,使用CLI程序taos, 登录进TDengine系统, 使用命令: - - ``` - CREATE DNODE "h2.taos.com:6030"; - ``` - - 将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**,否则出错。请注意将示例的“h2.taos.com:6030" 替换为你自己第一个节点的End Point - -6. 使用命令 - - ``` - SHOW DNODES; - ``` - - 查看新节点是否被成功加入。 - -按照上述步骤可以源源不断的将新的节点加入到集群。 - -**提示:** - -- firstEp, secondEp这两个参数仅仅在该节点第一次加入集群时有作用,加入集群后,该节点会保存最新的mnode的End Point列表,不再依赖这两个参数。 -- 两个没有配置first, second参数的dnode启动后,会独立运行起来。这个时候,无法将其中一个节点加入到另外一个节点,形成集群。**无法将两个独立的集群合并成为新的集群**。 - -##节点管理 - -###添加节点 -执行CLI程序taos, 使用root账号登录进系统, 执行: -``` -CREATE DNODE "fqdn:port"; -``` -将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**,否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置,缺省是自动获取。 - -###删除节点 -执行CLI程序taos, 使用root账号登录进TDengine系统,执行: -``` -DROP DNODE "fqdn:port"; -``` -其中fqdn是被删除的节点的FQDN,port是其对外服务器的端口号 - -###查看节点 -执行CLI程序taos,使用root账号登录进TDengine系统,执行: -``` -SHOW DNODES; -``` -它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等),vnode数目,还未使用的vnode数目等信息。在添加或删除一个节点后,可以使用该命令查看。 - -如果集群配置了Arbitrator,那么它也会在这个节点列表中显示出来,其role列的值会是“arb”。 - -###查看虚拟节点组 - -为充分利用多核技术,并提供scalability,数据需要分片处理。因此TDengine会将一个DB的数据切分成多份,存放在多个vnode里。这些vnode可能分布在多个dnode里,这样就实现了水平扩展。一个vnode仅仅属于一个DB,但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况,自动进行分配的,无需任何人工干预。 - -执行CLI程序taos,使用root账号登录进TDengine系统,执行: -``` -SHOW VGROUPS; -``` -##高可用性 -TDengine通过多副本的机制来提供系统的高可用性。副本数是与DB关联的,一个集群里可以有多个DB,根据运营的需求,每个DB可以配置不同的副本数。创建数据库时,通过参数replica 指定副本数(缺省为1)。如果副本数为1,系统的可靠性无法保证,只要数据所在的节点宕机,就将无法提供服务。集群的节点数必须大于等于副本数,否则创建表时将返回错误“more dnodes are needed"。比如下面的命令将创建副本数为3的数据库demo: -``` -CREATE DATABASE demo replica 3; -``` -一个DB里的数据会被切片分到多个vnode group,vnode group里的vnode数目就是DB的副本数,同一个vnode group里各vnode的数据是完全一致的。为保证高可用性,vnode group里的vnode一定要分布在不同的dnode里(实际部署时,需要在不同的物理机上),只要一个vgroup里超过半数的vnode处于工作状态,这个vgroup就能正常的对外服务。 - -一个dnode里可能有多个DB的数据,因此一个dnode离线时,可能会影响到多个DB。如果一个vnode group里的一半或一半以上的vnode不工作,那么该vnode group就无法对外服务,无法插入或读取数据,这样会影响到它所属的DB的一部分表的d读写操作。 - -因为vnode的引入,无法简单的给出结论:“集群中过半dnode工作,集群就应该工作”。但是对于简单的情形,很好下结论。比如副本数为3,只有三个dnode,那如果仅有一个节点不工作,整个集群还是可以正常工作的,但如果有两个节点不工作,那整个集群就无法正常工作了。 - -##Mnode的高可用 -TDengine集群是由mnode (taosd的一个模块,逻辑节点) 负责管理的,为保证mnode的高可用,可以配置多个mnode副本,副本数由系统配置参数numOfMnodes决定,有效范围为1-3。为保证元数据的强一致性,mnode副本之间是通过同步的方式进行数据复制的。 - -一个集群有多个dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下,哪个dnode可以作为mnode呢?这是完全由系统根据整个系统资源情况,自动指定的。用户可通过CLI程序taos,在TDengine的console里,执行如下命令: -``` -SHOW MNODES; -``` -来查看mnode列表,该列表将列出mnode所处的dnode的End Point和角色(master, slave, unsynced 或offline)。 -当集群中第一个节点启动时,该节点一定会运行一个mnode实例,否则该dnode无法正常工作,因为一个系统是必须有至少一个mnode的。如果numOfMnodes配置为2,启动第二个dnode时,该dnode也将运行一个mnode实例。 - -为保证mnode服务的高可用性,numOfMnodes必须设置为2或更大。因为mnode保存的元数据必须是强一致的,如果numOfMnodes大于2,复制参数quorum自动设为2,也就是说,至少要保证有两个副本写入数据成功,才通知客户端应用写入成功。 - -##负载均衡 - -有三种情况,将触发负载均衡,而且都无需人工干预。 - -- 当一个新节点添加进集群时,系统将自动触发负载均衡,一些节点上的数据将被自动转移到新节点上,无需任何人工干预。 -- 当一个节点从集群中移除时,系统将自动把该节点上的数据转移到其他节点,无需任何人工干预。 -- 如果一个节点过热(数据量过大),系统将自动进行负载均衡,将该节点的一些vnode自动挪到其他节点。 - -当上述三种情况发生时,系统将启动一各个节点的负载计算,从而决定如何挪动。 - -##节点离线处理 -如果一个节点离线,TDengine集群将自动检测到。有如下两种情况: -- 改节点离线超过一定时间(taos.cfg里配置参数offlineThreshold控制时长),系统将自动把该节点删除,产生系统报警信息,触发负载均衡流程。如果该被删除的节点重现上线时,它将无法加入集群,需要系统管理员重新将其添加进集群才会开始工作。 -- 离线后,在offlineThreshold的时长内重新上线,系统将自动启动数据恢复流程,等数据完全恢复后,该节点将开始正常工作。 - -##Arbitrator的使用 - -如果副本数为偶数,当一个vnode group里一半vnode不工作时,是无法从中选出master的。同理,一半mnode不工作时,是无法选出mnode的master的,因为存在“split brain”问题。为解决这个问题,TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作,但只简单的负责网络连接,不处理任何数据插入或访问。只要包含arbitrator在内,超过半数的vnode或mnode工作,那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形,如果一个节点A离线,但另外一个节点B正常,而且能连接到arbitrator, 那么节点B就能正常工作。 - -TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求,只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号,缺省是6030。配置每个taosd实例时,可以在配置文件taos.cfg里将参数arbitrator设置为Arbitrator的End Point。如果该参数配置了,当副本数为偶数数,系统将自动连接配置的Arbitrator。 - -在配置了Arbitrator的情况下,它也会显示在“show dnodes;”指令给出的节点列表中。 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 4aefc2f44930513c670719c5cad0da0cd0a643e5..83b70ed9f89e54ea336a4d066ab3d32bcfdd9c8a 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -29,12 +29,18 @@ # number of threads per CPU core # numOfThreadsPerCore 1.0 +# number of threads to commit cache data +# numOfCommitThreads 4 + # the proportion of total CPU cores available for query processing # 2.0: the query threads will be set to double of the CPU cores. # 1.0: all CPU cores are available for query processing [default]. # 0.5: only half of the CPU cores are available for query. # 0.0: only one core available. -# tsRatioOfQueryCores 1.0 +# ratioOfQueryCores 1.0 + +# the last_row/first/last aggregator will not change the original column name in the result fields +# keepColumnName 0 # number of management nodes in the system # numOfMnodes 3 @@ -262,11 +268,14 @@ # maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden # maxBinaryDisplayWidth 30 -# enable/disable telemetry reporting -# telemetryReporting 1 - # enable/disable stream (continuous query) # stream 1 # in retrieve blocking model, only in 50% query threads will be used in query processing in dnode # retrieveBlockingModel 0 + +# the maximum allowed query buffer size in MB during query processing for each data node +# -1 no limit (default) +# 0 no query allowed, queries are disabled +# queryBufferSize -1 + diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 3d57ece2adf4e8efd3338f52787d5d9424016e78..d24502a1cb8e69ddaf3989a89e51cc07dfb55f00 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -26,7 +26,6 @@ else ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 431093be950e2c2bb199ed294b57641290fe6221..36870b2ebe49d45390d7b5ce18f6984b9e8e2ac2 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -47,8 +47,10 @@ cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script +cp ${compile_dir}/../packaging/tools/startPre.sh ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/../packaging/tools/set_core.sh ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin -cp ${compile_dir}/build/bin/taosdemox ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 44ccafd04460c8beef0a4b2f5ae715033328f807..629f8f9fd6a4167db6f30d29646263710602693a 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -1,19 +1,24 @@ -FROM ubuntu:20.04 +FROM ubuntu:18.04 WORKDIR /root -ARG version -RUN echo $version -COPY tdengine.tar.gz /root/ -RUN tar -zxf tdengine.tar.gz -WORKDIR /root/TDengine-server-$version/ -RUN /bin/bash install.sh -e no +ARG pkgFile +ARG dirName +RUN echo ${pkgFile} +RUN echo ${dirName} +COPY ${pkgFile} /root/ +RUN tar -zxf ${pkgFile} +WORKDIR /root/${dirName}/ +RUN /bin/bash install.sh -e no +RUN apt-get clean && apt-get update && apt-get install -y locales +RUN locale-gen en_US.UTF-8 ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LC_CTYPE=en_US.UTF-8 ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en ENV LC_ALL=en_US.UTF-8 + EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 CMD ["taosd"] VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh new file mode 100755 index 0000000000000000000000000000000000000000..c846e8345dd65f645d5c00fcf047582b709138e7 --- /dev/null +++ b/packaging/docker/dockerManifest.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e +#set -x + +# dockerbuild.sh +# -n [version number] +# -p [xxxx] + +# set parameters by default value +verNumber="" +passWord="" + +while getopts "hn:p:" arg +do + case $arg in + n) + #echo "verNumber=$OPTARG" + verNumber=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "verNumber=${verNumber}" + +#docker manifest create -a tdengine/tdengine:${verNumber} tdengine/tdengine-amd64:${verNumber} tdengine/tdengine-aarch64:${verNumber} tdengine/tdengine-aarch32:${verNumber} +docker manifest create -a tdengine/tdengine:latest tdengine/tdengine-amd64:latest tdengine/tdengine-aarch64:latest tdengine/tdengine-aarch32:latest + +docker login -u tdengine -p ${passWord} #replace the docker registry username and password + +docker manifest push tdengine/tdengine:latest + +# how set latest version ??? diff --git a/packaging/docker/dockerbuild-aarch64.sh b/packaging/docker/dockerbuild-aarch64.sh deleted file mode 100755 index 795eaed5491834cf5601e567440a314d6e1f9629..0000000000000000000000000000000000000000 --- a/packaging/docker/dockerbuild-aarch64.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -x -docker build --rm -f "Dockerfile" -t tdengine/tdengine-aarch64:$1 "." --build-arg version=$1 -docker login -u tdengine -p $2 #replace the docker registry username and password -docker push tdengine/tdengine-aarch64:$1 diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index 48e2f7ead63356e8210d03beff2add6796558f5b..b7991465b0940c86ac6d620d498901ab5e1c9ac2 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -1,5 +1,63 @@ #!/bin/bash -set -x -docker build --rm -f "Dockerfile" -t tdengine/tdengine:$1 "." --build-arg version=$1 -docker login -u tdengine -p $2 #replace the docker registry username and password -docker push tdengine/tdengine:$1 +set -e +#set -x + +# dockerbuild.sh +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -f [pkg file] +# -n [version number] +# -p [password for docker hub] + +# set parameters by default value +cpuType=amd64 +verNumber="" +passWord="" +pkgFile="" + +while getopts "hc:n:p:f:" arg +do + case $arg in + c) + #echo "cpuType=$OPTARG" + cpuType=$(echo $OPTARG) + ;; + n) + #echo "verNumber=$OPTARG" + verNumber=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + f) + #echo "pkgFile=$OPTARG" + pkgFile=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo " -f [pkg file] " + echo " -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "cpuType=${cpuType} verNumber=${verNumber} pkgFile=${pkgFile} " +echo "$(pwd)" +echo "====NOTES: ${pkgFile} must be in the same directory as dockerbuild.sh====" + +dirName=${pkgFile%-Linux*} +#echo "dirName=${dirName}" + +docker build --rm -f "Dockerfile" -t tdengine/tdengine-${cpuType}:${verNumber} "." --build-arg pkgFile=${pkgFile} --build-arg dirName=${dirName} +docker login -u tdengine -p ${passWord} #replace the docker registry username and password +docker push tdengine/tdengine-${cpuType}:${verNumber} + +# set this version to latest version +docker tag tdengine/tdengine-${cpuType}:${verNumber} tdengine/tdengine-${cpuType}:latest +docker push tdengine/tdengine-${cpuType}:latest diff --git a/packaging/docker/dockerbuildi.sh b/packaging/docker/dockerbuildi.sh new file mode 100755 index 0000000000000000000000000000000000000000..a0a954e30fe9c3637abe4d219001235d793466e0 --- /dev/null +++ b/packaging/docker/dockerbuildi.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# + +set -e +#set -x + +# dockerbuild.sh +# -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] +# -n [version number] +# -p [password for docker hub] + +# set parameters by default value +cpuType=aarch64 +verNumber="" +passWord="" + +while getopts "hc:n:p:f:" arg +do + case $arg in + c) + #echo "cpuType=$OPTARG" + cpuType=$(echo $OPTARG) + ;; + n) + #echo "verNumber=$OPTARG" + verNumber=$(echo $OPTARG) + ;; + p) + #echo "passWord=$OPTARG" + passWord=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -c [aarch32 | aarch64 | amd64 | x86 | mips64 ...] " + echo " -n [version number] " + echo " -p [password for docker hub] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +pkgFile=TDengine-server-${verNumber}-Linux-${cpuType}.tar.gz + +echo "cpuType=${cpuType} verNumber=${verNumber} pkgFile=${pkgFile} " + +scriptDir=`pwd` +pkgDir=$scriptDir/../../release/ + +cp -f ${pkgDir}/${pkgFile} . + +./dockerbuild.sh -c ${cpuType} -f ${pkgFile} -n ${verNumber} -p ${passWord} + +rm -f ${pkgFile} diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 6f012aa80eac67ba42e7ca561c95ef6d905c2100..92c917cb3d6d4cd5f41441c9ca75a742aa3641b6 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -9,6 +9,7 @@ Summary: tdengine from taosdata Group: Application/Database License: AGPL URL: www.taosdata.com +AutoReqProv: no #BuildRoot: %_topdir/BUILDROOT BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root @@ -55,10 +56,12 @@ cp %{_compiledir}/../packaging/cfg/taos.cfg %{buildroot}%{homepath}/cfg cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/script +cp %{_compiledir}/../packaging/tools/startPre.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../packaging/tools/set_core.sh %{buildroot}%{homepath}/bin +cp %{_compiledir}/../packaging/tools/taosd-dump-cfg.gdb %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin -cp %{_compiledir}/build/bin/taosdemox %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include @@ -136,7 +139,6 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index d98ed2185f014356fcf6e954bd717be98a960d16..dca3dd2ff623672eb85b3de72bcc34e0ea5e3d8a 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -147,8 +147,8 @@ done #echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" -function kill_taosd() { - pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo} kill -9 $pid || : fi @@ -168,6 +168,10 @@ function install_main_path() { if [ "$verMode" == "cluster" ]; then ${csudo} mkdir -p ${nginx_dir} fi + + if [[ -e ${script_dir}/email ]]; then + ${csudo} cp ${script_dir}/email ${install_main_dir}/ ||: + fi } function install_bin() { @@ -175,7 +179,6 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : @@ -187,7 +190,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : - [ -x ${install_main_dir}/bin/taosdemox ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemox ${bin_link_dir}/taosdemox || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : @@ -604,9 +606,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" - #${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/setDelay.sh' >> ${taosd_service_config}" - #${csudo} bash -c "echo 'ExecStartPost=/usr/local/taos/bin/resetDelay.sh' >> ${taosd_service_config}" - #${csudo} bash -c "echo 'ExecStopPost=/usr/local/taos/bin/resetDelay.sh' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" @@ -681,7 +681,7 @@ function install_service() { install_service_on_sysvinit else # must manual stop taosd - kill_taosd + kill_process taosd fi } @@ -750,9 +750,22 @@ function update_TDengine() { elif ((${service_mod}==1)); then ${csudo} service taosd stop || : else - kill_taosd + kill_process taosd + fi + sleep 1 + fi + + if [ "$verMode" == "cluster" ]; then + if pidof nginx &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop nginxd || : + elif ((${service_mod}==1)); then + ${csudo} service nginxd stop || : + else + kill_process nginx fi sleep 1 + fi fi install_main_path diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index dd116e9bfb0472b661321e9af786521300ed2f74..0a0a6633e376d084532abb5f490917abd1a173f2 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -21,7 +21,7 @@ else cd ${script_dir} script_dir="$(pwd)" data_dir="/var/lib/taos" - log_dir="~/TDengineLog" + log_dir=~/TDengine/log fi log_link_dir="/usr/local/taos/log" @@ -86,7 +86,6 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : if [ "$osType" != "Darwin" ]; then ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : fi ${csudo} rm -f ${bin_link_dir}/rmtaos || : @@ -98,7 +97,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : - [ -x ${install_main_dir}/bin/taosdemox ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemox ${bin_link_dir}/taosdemox || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || : diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index 04fd23d5abb2874d0b3a68b937be529bd2c91a59..8d7463366ff46bcae2822ee3e76dbc9b588f2a89 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -86,7 +86,6 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/power || : if [ "$osType" != "Darwin" ]; then ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdemox || : ${csudo} rm -f ${bin_link_dir}/powerdump || : fi ${csudo} rm -f ${bin_link_dir}/rmpower || : @@ -98,7 +97,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : - [ -x ${install_main_dir}/bin/powerdemox ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemox ${bin_link_dir}/powerdemox || : [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || : fi [ -x ${install_main_dir}/bin/remove_client_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_power.sh ${bin_link_dir}/rmpower || : diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index b14d5d400b9332c52f1559dcfef72a287d9fc385..ba6ace400935c10caadd9426c0701c16b4f86baa 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -146,8 +146,8 @@ done #echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" -function kill_powerd() { - pid=$(ps -ef | grep "powerd" | grep -v "grep" | awk '{print $2}') +function kill_process() { + pid=$(ps -ef | grep "$1" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then ${csudo} kill -9 $pid || : fi @@ -174,7 +174,6 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerd || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdemox || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -185,7 +184,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : [ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd || : [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : - [ -x ${install_main_dir}/bin/powerdemox ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemox ${bin_link_dir}/powerdemox || : [ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : @@ -578,6 +576,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${powerd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${powerd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/powerd' >> ${powerd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/power/bin/startPre.sh' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${powerd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${powerd_service_config}" @@ -651,7 +650,7 @@ function install_service() { install_service_on_sysvinit else # must manual stop powerd - kill_powerd + kill_process powerd fi } @@ -720,9 +719,21 @@ function update_PowerDB() { elif ((${service_mod}==1)); then ${csudo} service powerd stop || : else - kill_powerd + kill_process powerd fi sleep 1 + fi + if [ "$verMode" == "cluster" ]; then + if pidof nginx &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop nginxd || : + elif ((${service_mod}==1)); then + ${csudo} service nginxd stop || : + else + kill_process nginx + fi + sleep 1 + fi fi install_main_path diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 831012851ad70d05080bfae161c0b925d5215ae9..1fd0e943b155d0c96c26c3fdcec7061bd25197e4 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -24,7 +24,7 @@ data_dir="/var/lib/taos" if [ "$osType" != "Darwin" ]; then log_dir="/var/log/taos" else - log_dir="~/TDengineLog" + log_dir=~/TDengine/log fi data_link_dir="/usr/local/taos/data" @@ -149,10 +149,12 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} cp -r ${binary_dir}/build/bin/* ${install_main_dir}/bin + ${csudo} cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin if [ "$osType" != "Darwin" ]; then - ${csudo} cp -r ${script_dir}/remove.sh ${install_main_dir}/bin + ${csudo} cp -r ${script_dir}/remove.sh ${install_main_dir}/bin ${csudo} cp -r ${script_dir}/set_core.sh ${install_main_dir}/bin + ${csudo} cp -r ${script_dir}/startPre.sh ${install_main_dir}/bin else ${csudo} cp -r ${script_dir}/remove_client.sh ${install_main_dir}/bin fi @@ -178,7 +180,9 @@ function install_bin() { function install_lib() { # Remove links ${csudo} rm -f ${lib_link_dir}/libtaos.* || : - ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + if [ "$osType" != "Darwin" ]; then + ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : + fi if [ "$osType" != "Darwin" ]; then ${csudo} cp ${binary_dir}/build/lib/libtaos.so.${verNumber} ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* @@ -190,12 +194,14 @@ function install_lib() { ${csudo} ln -sf ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so fi else - ${csudo} cp ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* - ${csudo} ln -sf ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.1.dylib + ${csudo} cp -Rf ${binary_dir}/build/lib/libtaos.* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* + ${csudo} ln -sf ${install_main_dir}/driver/libtaos.1.dylib ${lib_link_dir}/libtaos.1.dylib ${csudo} ln -sf ${lib_link_dir}/libtaos.1.dylib ${lib_link_dir}/libtaos.dylib fi - ${csudo} ldconfig + if [ "$osType" != "Darwin" ]; then + ${csudo} ldconfig + fi } function install_header() { @@ -326,6 +332,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 00dfcb75590a50d23180e5eb2b6ad38f28924bba..30e9fa51a7d9c4a98d2c8f300287ebd242fecd74 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,8 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdemox ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \ + ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else @@ -54,7 +55,11 @@ else fi header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" -cfg_dir="${top_dir}/packaging/cfg" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi install_files="${script_dir}/install_client.sh" diff --git a/packaging/tools/makeclient_power.sh b/packaging/tools/makeclient_power.sh index 509df31297d5b442a05516b37a1b82572ae4f610..181536b7f19d252164201291d9c37cade6cf3490 100755 --- a/packaging/tools/makeclient_power.sh +++ b/packaging/tools/makeclient_power.sh @@ -54,7 +54,11 @@ else fi header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" -cfg_dir="${top_dir}/packaging/cfg" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi install_files="${script_dir}/install_client_power.sh" @@ -77,10 +81,10 @@ if [ "$osType" != "Darwin" ]; then cp ${build_dir}/bin/taos ${install_dir}/bin/power cp ${script_dir}/remove_power.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo - cp ${build_dir}/bin/taosdemox ${install_dir}/bin/powerdemox cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin fi else cp ${bin_files} ${install_dir}/bin diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 0b4659a9112f933a5ba9b91f52f3df203d18a2e7..36b1fe5bd88950f69a56e84d98fba9c4dae0cf05 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,12 +36,18 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdemox ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator\ + ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" fi lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" -cfg_dir="${top_dir}/packaging/cfg" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi + install_files="${script_dir}/install.sh" nginx_dir="${code_dir}/../../enterprise/src/plugins/web" diff --git a/packaging/tools/makepkg_power.sh b/packaging/tools/makepkg_power.sh index ba57fe5e9687b32b73a0a3ff4d48807f04a0202d..554e7884b1c3db69acd3ba0e3234e468b1d31c79 100755 --- a/packaging/tools/makepkg_power.sh +++ b/packaging/tools/makepkg_power.sh @@ -36,12 +36,17 @@ fi # strip ${build_dir}/bin/taos # bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${script_dir}/remove_power.sh" #else -# bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh ${script_dir}/set_core.sh" +# bin_files="${build_dir}/bin/powerd ${build_dir}/bin/power ${build_dir}/bin/powerdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove_power.sh\ +# ${script_dir}/set_core.sh ${script_dir}/startPre.sh ${script_dir}/taosd-dump-cfg.gdb" #fi lib_files="${build_dir}/lib/libtaos.so.${version}" header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" -cfg_dir="${top_dir}/packaging/cfg" +if [ "$verMode" == "cluster" ]; then + cfg_dir="${top_dir}/../enterprise/packaging/cfg" +else + cfg_dir="${top_dir}/packaging/cfg" +fi install_files="${script_dir}/install_power.sh" nginx_dir="${code_dir}/../../enterprise/src/plugins/web" @@ -77,11 +82,12 @@ else cp ${build_dir}/bin/taosd ${install_dir}/bin/powerd cp ${script_dir}/remove_power.sh ${install_dir}/bin cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo - cp ${build_dir}/bin/taosdemox ${install_dir}/bin/powerdemox cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump cp ${build_dir}/bin/tarbitrator ${install_dir}/bin cp ${script_dir}/set_core.sh ${install_dir}/bin cp ${script_dir}/get_client.sh ${install_dir}/bin + cp ${script_dir}/startPre.sh ${install_dir}/bin + cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin fi chmod a+x ${install_dir}/bin/* || : diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 6bfbf33fd1dd8011e0fbd77322b9f104a0ab8c30..8665b3fec3a392b3dcae8c6a197625ba85ed953b 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -96,7 +96,6 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -107,7 +106,6 @@ function install_bin() { [ -x ${bin_dir}/taos ] && ${csudo} ln -s ${bin_dir}/taos ${bin_link_dir}/taos || : [ -x ${bin_dir}/taosd ] && ${csudo} ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : [ -x ${bin_dir}/taosdemo ] && ${csudo} ln -s ${bin_dir}/taosdemo ${bin_link_dir}/taosdemo || : - [ -x ${bin_dir}/taosdemox ] && ${csudo} ln -s ${bin_dir}/taosdemox ${bin_link_dir}/taosdemox || : [ -x ${bin_dir}/taosdump ] && ${csudo} ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : } @@ -406,6 +404,7 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Service]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Type=simple' >> ${taosd_service_config}" ${csudo} bash -c "echo 'ExecStart=/usr/bin/taosd' >> ${taosd_service_config}" + ${csudo} bash -c "echo 'ExecStartPre=/usr/local/taos/bin/startPre.sh' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${taosd_service_config}" ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${taosd_service_config}" diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 8d96ef851c9812dcf7764dd1150350721bd2ec6e..9241f01efaeb892afc020dc239e5e80eebc8bdd6 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -72,7 +72,6 @@ function clean_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : @@ -121,7 +120,7 @@ function clean_service_on_systemd() { if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/${nginx_service_name}.service" - if [ -d ${bin_dir}/web ]; then + if [ -d ${install_nginxd_dir} ]; then if systemctl is-active --quiet ${nginx_service_name}; then echo "Nginx for TDengine is running, stopping it..." ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null @@ -214,10 +213,10 @@ fi if echo $osinfo | grep -qwi "ubuntu" ; then # echo "this is ubuntu system" - ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : + ${csudo} dpkg --force-all -P tdengine || : elif echo $osinfo | grep -qwi "debian" ; then # echo "this is debian system" - ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : + ${csudo} dpkg --force-all -P tdengine || : elif echo $osinfo | grep -qwi "centos" ; then # echo "this is centos system" ${csudo} rpm -e --noscripts tdengine || : diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index e84cdd2620f7e83ea12af7500c2385892d7072fa..7579162dc60e290754e71ed6a71c10cfaee5537b 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -38,7 +38,6 @@ function clean_bin() { # Remove link ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdemox || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/set_core || : diff --git a/packaging/tools/remove_client_power.sh b/packaging/tools/remove_client_power.sh index 1842e86a5b2c55b6c2aac02e6150bc9eaa5836a6..580c46e2077d7f21e06d4d4a8f69dcd5b6bbf51d 100755 --- a/packaging/tools/remove_client_power.sh +++ b/packaging/tools/remove_client_power.sh @@ -38,7 +38,6 @@ function clean_bin() { # Remove link ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdemox || : ${csudo} rm -f ${bin_link_dir}/powerdump || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/set_core || : diff --git a/packaging/tools/remove_power.sh b/packaging/tools/remove_power.sh index 59073105de9828bc1255d269bdc773d613500e5a..816869cf444d8001e0c0aae30840d2c40a9e6af4 100755 --- a/packaging/tools/remove_power.sh +++ b/packaging/tools/remove_power.sh @@ -72,7 +72,6 @@ function clean_bin() { ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerd || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : - ${csudo} rm -f ${bin_link_dir}/powerdemox || : ${csudo} rm -f ${bin_link_dir}/powerdump || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : diff --git a/packaging/tools/startPre.sh b/packaging/tools/startPre.sh new file mode 100644 index 0000000000000000000000000000000000000000..3c16a5a938bdf3d7dc36c0a79f46e9c8f32b222b --- /dev/null +++ b/packaging/tools/startPre.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# if enable core dump, set start count to 3, disable core dump, set start count to 20. +# set -e +# set -x + +taosd=/etc/systemd/system/taosd.service +line=`grep StartLimitBurst ${taosd}` +num=${line##*=} +#echo "burst num: ${num}" + +startSeqFile=/usr/local/taos/.startSeq +recordFile=/usr/local/taos/.startRecord + +startSeq=0 + +if [[ ! -e ${startSeqFile} ]]; then + startSeq=0 +else + startSeq=$(cat ${startSeqFile}) +fi + +nextSeq=`expr $startSeq + 1` +echo "${nextSeq}" > ${startSeqFile} + +curTime=$(date "+%Y-%m-%d %H:%M:%S") +echo "startSeq:${startSeq} startPre.sh exec ${curTime}, burstCnt:${num}" >> ${recordFile} + + +coreFlag=`ulimit -c` +echo "coreFlag: ${coreFlag}" >> ${recordFile} + +if [ ${coreFlag} = "0" ];then + #echo "core is 0" + if [ ${num} != "20" ];then + sed -i "s/^.*StartLimitBurst.*$/StartLimitBurst=20/" ${taosd} + systemctl daemon-reload + echo "modify burst count from ${num} to 20" >> ${recordFile} + fi +fi + +if [ ${coreFlag} = "unlimited" ];then + #echo "core is unlimited" + if [ ${num} != "3" ];then + sed -i "s/^.*StartLimitBurst.*$/StartLimitBurst=3/" ${taosd} + systemctl daemon-reload + echo "modify burst count from ${num} to 3" >> ${recordFile} + fi +fi + diff --git a/packaging/tools/taosd-dump-cfg.gdb b/packaging/tools/taosd-dump-cfg.gdb new file mode 100644 index 0000000000000000000000000000000000000000..9774ccf82283817edea5f49b59e0c6cb6f529577 --- /dev/null +++ b/packaging/tools/taosd-dump-cfg.gdb @@ -0,0 +1,144 @@ +# Usage: +# sudo gdb -x ./taosd-dump-cfg.gdb + +define attach_pidof + if $argc != 1 + help attach_pidof + else + shell echo -e "\ +set \$PID = "$(echo $(pidof $arg0) 0 | cut -d " " -f 1)"\n\ +if \$PID > 0\n\ + attach "$(pidof -s $arg0)"\n\ +else\n\ + print \"Process '"$arg0"' not found\"\n\ +end" > /tmp/gdb.pidof + source /tmp/gdb.pidof + end +end + +document attach_pidof +Attach to process by name +Usage: attach_pidof PROG_NAME +end + +set $TAOS_CFG_VTYPE_INT8 = 0 +set $TAOS_CFG_VTYPE_INT16 = 1 +set $TAOS_CFG_VTYPE_INT32 = 2 +set $TAOS_CFG_VTYPE_FLOAT = 3 +set $TAOS_CFG_VTYPE_STRING = 4 +set $TAOS_CFG_VTYPE_IPSTR = 5 +set $TAOS_CFG_VTYPE_DIRECTORY = 6 + +set $TSDB_CFG_CTYPE_B_CONFIG = 1U +set $TSDB_CFG_CTYPE_B_SHOW = 2U +set $TSDB_CFG_CTYPE_B_LOG = 4U +set $TSDB_CFG_CTYPE_B_CLIENT = 8U +set $TSDB_CFG_CTYPE_B_OPTION = 16U +set $TSDB_CFG_CTYPE_B_NOT_PRINT = 32U + +set $TSDB_CFG_PRINT_LEN = 53 + +define print_blank + if $argc == 1 + set $blank_len = $arg0 + while $blank_len > 0 + printf "%s", " " + set $blank_len = $blank_len - 1 + end + end +end + +define dump_cfg + if $argc != 1 + help dump_cfg + else + set $blen = $TSDB_CFG_PRINT_LEN - (int)strlen($arg0.option) + if $blen < 0 + $blen = 0 + end + #printf "%s: %d\n", "******blen: ", $blen + printf "%s: ", $arg0.option + print_blank $blen + + if $arg0.valType == $TAOS_CFG_VTYPE_INT8 + printf "%d\n", *((int8_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_INT16 + printf "%d\n", *((int16_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_INT32 + printf "%d\n", *((int32_t *) $arg0.ptr) + else + if $arg0.valType == $TAOS_CFG_VTYPE_FLOAT + printf "%f\n", *((float *) $arg0.ptr) + else + printf "%s\n", $arg0.ptr + end + end + end + end + end +end + +document dump_cfg +Dump a cfg entry +Usage: dump_cfg cfg +end + +set pagination off + +attach_pidof taosd + +set $idx=0 +#print tsGlobalConfigNum +#set $end=$1 +set $end=tsGlobalConfigNum + +p "*=*=*=*=*=*=*=*=*= taos global config:" +#while ($idx .lt. $end) +while ($idx < $end) + # print tsGlobalConfig[$idx].option + set $cfg = tsGlobalConfig[$idx] + set $tsce = tscEmbedded +# p "1" + if ($tsce == 0) + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_CLIENT) + end + else + if $cfg.cfgType & $TSDB_CFG_CTYPE_B_NOT_PRINT + else + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_SHOW) + else + dump_cfg $cfg + end + end + end + + set $idx=$idx+1 +end + +set $idx=0 + +p "*=*=*=*=*=*=*=*=*= taos local config:" +while ($idx < $end) + set $cfg = tsGlobalConfig[$idx] + set $tsce = tscEmbedded + if ($tsce == 0) + if !($cfg.cfgType & $TSDB_CFG_CTYPE_B_CLIENT) + end + else + if $cfg.cfgType & $TSDB_CFG_CTYPE_B_NOT_PRINT + else + if ($cfg.cfgType & $TSDB_CFG_CTYPE_B_SHOW) + else + dump_cfg $cfg + end + end + end + + set $idx=$idx+1 +end + +detach + +quit diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 102fea6b9e3f8892d8e8af26c6ee54bdaa948fb8..88628b4db6cfafa2a9815489313ced1345f2b600 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.14.0' +version: '2.0.18.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.14.0 + - usr/lib/libtaos.so.2.0.18.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 04bc61ed9e1ca0eb0480bc13d1a9731827e0239e..8cc5cee3b51675b2d42ad62c442b2b030e802cbb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,10 +1,11 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) # Base compile ADD_SUBDIRECTORY(os) ADD_SUBDIRECTORY(common) ADD_SUBDIRECTORY(util) +ADD_SUBDIRECTORY(tfs) ADD_SUBDIRECTORY(rpc) ADD_SUBDIRECTORY(client) ADD_SUBDIRECTORY(query) @@ -18,6 +19,6 @@ ADD_SUBDIRECTORY(tsdb) ADD_SUBDIRECTORY(wal) ADD_SUBDIRECTORY(cq) ADD_SUBDIRECTORY(dnode) -#ADD_SUBDIRECTORY(connector/odbc) +ADD_SUBDIRECTORY(connector/odbc) ADD_SUBDIRECTORY(connector/jdbc) diff --git a/src/balance/CMakeLists.txt b/src/balance/CMakeLists.txt index fdc43ea32fd8913b984005f30621a801e50c45d4..967635e52ce20761dbd674a380563deeeb9af189 100644 --- a/src/balance/CMakeLists.txt +++ b/src/balance/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) @@ -8,6 +8,4 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) - ADD_LIBRARY(balance ${SRC}) -ENDIF () +ADD_LIBRARY(balance ${SRC}) diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h index 8f54b66028c5ddc859861bb78e0d790f7d3e5507..74a761299da9bb0816869dae3fd49a6afb6350bd 100644 --- a/src/balance/inc/bnThread.h +++ b/src/balance/inc/bnThread.h @@ -24,7 +24,7 @@ extern "C" { int32_t bnInitThread(); void bnCleanupThread(); void bnNotify(); -void bnStartTimer(int64_t mseconds); +void bnStartTimer(int32_t mseconds); #ifdef __cplusplus } diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 3e1d5eda763c99afb8a0c5dcd3f08996eec80337..3055f77e81021c4877d97147033fc34f4c525df9 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -30,7 +30,7 @@ #include "mnodeVgroup.h" extern int64_t tsDnodeRid; -extern int64_t tsSdbRid; +extern int32_t tsSdbRid; static SBnMgmt tsBnMgmt; static void bnMonitorDnodeModule(); @@ -425,7 +425,7 @@ static bool bnMonitorVgroups() { while (1) { pIter = mnodeGetNextVgroup(pIter, &pVgroup); - if (pVgroup == NULL) break; + if (pVgroup == NULL || pVgroup->pDb == NULL) break; int32_t dbReplica = pVgroup->pDb->cfg.replications; int32_t vgReplica = pVgroup->numOfVnodes; @@ -721,4 +721,4 @@ int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnode mnodeDecDnodeRef(pDestDnode); return code; -} \ No newline at end of file +} diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c index cbc2c6218458a1ec194ff49f215b80a5195d2271..7d94df1c23ab7824dbada0423beec14530a2101c 100644 --- a/src/balance/src/bnScore.c +++ b/src/balance/src/bnScore.c @@ -271,23 +271,23 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = systemScore; + *(float *)pWrite = (float)systemScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = pDnode->customScore; + *(float *)pWrite = (float)pDnode->customScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)moduleScore; + *(float *)pWrite = (float)moduleScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)vnodeScore; + *(float *)pWrite = (float)vnodeScore; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore); + *(float *)pWrite = (float)(vnodeScore + moduleScore + pDnode->customScore + systemScore); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index 84f8694fca7248abb27529f5e8268dc0e08bf815..d07591ecd55a9a10613cc61de3a77ea85e022bc2 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -56,7 +56,7 @@ int32_t bnInitThread() { pthread_attr_destroy(&thattr); if (ret != 0) { - mError("failed to create balance thread since %s", strerror(errno)); + mError("failed to create balance thread since %s", strerror(ret)); return -1; } @@ -119,13 +119,13 @@ static void bnProcessTimer(void *handle, void *tmrId) { } } -void bnStartTimer(int64_t mseconds) { +void bnStartTimer(int32_t mseconds) { if (tsBnThread.stop) return; bool updateSoon = (mseconds != -1); if (updateSoon) { - mTrace("balance function will be called after %" PRId64 " ms", mseconds); - taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); + mTrace("balance function will be called after %d ms", mseconds); + taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer); } else { taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); } diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 4049c0d72908486d289631362c19e7caa25f99f8..fb43751b9e8fd715d538abb1198e1bdfd0a2e9ae 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) @@ -28,6 +28,28 @@ IF (TD_LINUX) ADD_SUBDIRECTORY(tests) +ELSEIF (TD_DARWIN) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) + + # set the static lib name + ADD_LIBRARY(taos_static STATIC ${SRC}) + TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m) + SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") + SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) + + # generate dynamic library (*.dylib) + ADD_LIBRARY(taos SHARED ${SRC}) + TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m) + SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) + + #set version of .dylib + #VERSION dylib version + #SOVERSION dylib version + #MESSAGE(STATUS "build version ${TD_VER_NUMBER}") + SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) + + ADD_SUBDIRECTORY(tests) + ELSEIF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) @@ -49,12 +71,12 @@ ELSEIF (TD_DARWIN) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux) ADD_LIBRARY(taos_static STATIC ${SRC}) - TARGET_LINK_LIBRARIES(taos_static trpc tutil pthread m) + TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m) SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static") # generate dynamic library (*.dylib) ADD_LIBRARY(taos SHARED ${SRC}) - TARGET_LINK_LIBRARIES(taos trpc tutil pthread m) + TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m) SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 06176451881b30ebe4c24bfcd478ceeb221a86de..581cd37cbd53cb87847fc5a13c88b03eb797d93a 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -38,7 +38,7 @@ typedef struct SLocalDataSource { tFilePage filePage; } SLocalDataSource; -typedef struct SLocalReducer { +typedef struct SLocalMerger { SLocalDataSource ** pLocalDataSrc; int32_t numOfBuffer; int32_t numOfCompleted; @@ -62,7 +62,7 @@ typedef struct SLocalReducer { bool discard; int32_t offset; // limit offset value bool orderPrjOnSTable; // projection query on stable -} SLocalReducer; +} SLocalMerger; typedef struct SRetrieveSupport { tExtMemBuffer ** pExtMemBuffer; // for build loser tree @@ -89,10 +89,10 @@ int32_t tscFlushTmpBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tF /* * create local reducer to launch the second-stage reduce process at client site */ -void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, +void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, SColumnModel *finalModel, SColumnModel *pFFModel, SSqlObj* pSql); -void tscDestroyLocalReducer(SSqlObj *pSql); +void tscDestroyLocalMerger(SSqlObj *pSql); int32_t tscDoLocalMerge(SSqlObj *pSql); diff --git a/src/client/inc/tscLog.h b/src/client/inc/tscLog.h index 5273a87ea0d0549420acf2b6679f50ce22159ebc..f25ec02bd8204d124477e6eba1070520b4d00c4e 100644 --- a/src/client/inc/tscLog.h +++ b/src/client/inc/tscLog.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSC_LOG_H -#define TDENGINE_TSC_LOG_H +#ifndef TDENGINE_TSCLOG_H +#define TDENGINE_TSCLOG_H #ifdef __cplusplus extern "C" { @@ -22,7 +22,7 @@ extern "C" { #include "tlog.h" -extern int32_t cDebugFlag; +extern uint32_t cDebugFlag; extern int8_t tscEmbedded; #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", tscEmbedded ? 255 : cDebugFlag, __VA_ARGS__); }} while(0) diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index 43c9b009bfdc537b033b7cffbe00bb673c627847..15ef54b7b13eb8463c54a7b979eb6d007560bb0e 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCJOINPROCESS_H -#define TDENGINE_TSCJOINPROCESS_H +#ifndef TDENGINE_TSCSUBQUERY_H +#define TDENGINE_TSCSUBQUERY_H #ifdef __cplusplus extern "C" { @@ -33,6 +33,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index); void tscHandleMasterJoinQuery(SSqlObj* pSql); int32_t tscHandleMasterSTableQuery(SSqlObj *pSql); +int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql); int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); @@ -47,9 +48,8 @@ void tscLockByThread(int64_t *lockedBy); void tscUnlockByThread(int64_t *lockedBy); - #ifdef __cplusplus } #endif -#endif // TDENGINE_TSCJOINPROCESS_H +#endif // TDENGINE_TSCSUBQUERY_H diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 8d82c65cee5cb26c3a4a37e6bbe6076dae28aaa5..3eba5d579bf05a8d6336b1a1212faa2cb920108a 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -83,6 +83,22 @@ typedef struct SJoinSupporter { SArray* pVgroupTables; } SJoinSupporter; + +typedef struct SMergeCtx { + SJoinSupporter* p; + int32_t idx; + SArray* res; + int8_t compared; +}SMergeCtx; + +typedef struct SMergeTsCtx { + SJoinSupporter* p; + STSBuf* res; + int64_t numOfInput; + int8_t compared; +}SMergeTsCtx; + + typedef struct SVgroupTableInfo { SVgroupInfo vgInfo; SArray* itemList; //SArray @@ -98,20 +114,19 @@ static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t sub return pCmd->pQueryInfo[subClauseIndex]; } -int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, - STableMeta* pTableMeta, STableDataBlocks** dataBlocks); -void tscDestroyDataBlock(STableDataBlocks* pDataBlock); +int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks); +void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset); void* tscDestroyBlockArrayList(SArray* pDataBlockList); -void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable); +void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta); int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock); -int32_t tscMergeTableDataBlocks(SSqlObj* pSql); -int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, +int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap); +int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, SName* pName, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList); /** @@ -124,6 +139,7 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo); bool tscIsTWAQuery(SQueryInfo* pQueryInfo); bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo); +bool tscGroupbyColumn(SQueryInfo* pQueryInfo); bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex); bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); @@ -133,8 +149,10 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); +bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); +bool tscQueryBlockInfo(SQueryInfo* pQueryInfo); -SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql); @@ -142,10 +160,6 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); -/* use for keep current db info temporarily, for handle table with db prefix */ -// todo remove it -void tscGetDBInfoFromTableFullName(char* tableId, char* db); - int tscAllocPayload(SSqlCmd* pCmd, int size); TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); @@ -156,7 +170,6 @@ SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_F SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index); TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); -void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); @@ -179,12 +192,14 @@ SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIn SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type, int16_t size); size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo); +void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex); SSqlExpr* tscSqlExprGet(SQueryInfo* pQueryInfo, int32_t index); int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy); void tscSqlExprInfoDestroy(SArray* pExprInfo); SColumn* tscColumnClone(const SColumn* src); +bool tscColumnExists(SArray* pColumnList, SColumnIndex* pColIndex); SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); SArray* tscColumnListClone(const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); @@ -215,8 +230,8 @@ SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo); -STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); +STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, + SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); @@ -225,7 +240,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscFreeVgroupTableInfo(SArray* pVgroupTables); -SArray* tscVgroupTableInfoClone(SArray* pVgroupTables); +SArray* tscVgroupTableInfoDup(SArray* pVgroupTables); void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); @@ -256,11 +271,11 @@ void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src); * @param pPrevSql * @return */ -SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd); +SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd); void registerSqlObj(SSqlObj* pSql); -SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql); +SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql); void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex); void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex); @@ -273,7 +288,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex); bool hasMoreVnodesToTry(SSqlObj *pSql); bool hasMoreClauseToTry(SSqlObj* pSql); -void tscFreeQueryInfo(SSqlCmd* pCmd); +void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); @@ -292,7 +307,7 @@ uint32_t tscGetTableMetaSize(STableMeta* pTableMeta); CChildTableMeta* tscCreateChildMeta(STableMeta* pTableMeta); uint32_t tscGetTableMetaMaxSize(); int32_t tscCreateTableMetaFromCChildMeta(STableMeta* pChild, const char* name); -STableMeta* tscTableMetaClone(STableMeta* pTableMeta); +STableMeta* tscTableMetaDup(STableMeta* pTableMeta); void* malloc_throw(size_t size); diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index c881a7a7634c774eee91c64fb41582a1a11683ca..a9dcd230a6db824f8cf40e05603c78bd95e0168b 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -24,10 +24,6 @@ extern "C" { #include "tstoken.h" #include "tsclient.h" -#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) - -#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) - /** * get the number of tags of this table * @param pTableMeta @@ -79,26 +75,6 @@ SSchema *tscGetTableColumnSchema(const STableMeta *pMeta, int32_t colIndex); */ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId); -/** - * check if the schema is valid or not, including following aspects: - * 1. number of columns - * 2. column types - * 3. column length - * 4. column names - * 5. total length - * - * @param pSchema - * @param numOfCols - * @return - */ -bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); - -/** - * get the schema for the "tbname" column. it is a built column - * @return - */ -SSchema tscGetTbnameColumnSchema(); - /** * create the table meta from the msg * @param pTableMetaMsg diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 900fab53a2f6c93933f50e11307d92ce7393960c..f233af35a3f08d0d766167dfcac46235b0480f4c 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -22,15 +22,15 @@ extern "C" { #include "os.h" +#include "qAggMain.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" -#include "tglobal.h" -#include "tsqlfunction.h" -#include "tutil.h" #include "tcache.h" +#include "tglobal.h" #include "tref.h" +#include "tutil.h" #include "qExecutor.h" #include "qSqlparser.h" @@ -39,7 +39,7 @@ extern "C" { // forward declaration struct SSqlInfo; -struct SLocalReducer; +struct SLocalMerger; // data source from sql string or from file enum { @@ -67,7 +67,7 @@ typedef struct CChildTableMeta { int32_t vgId; STableId id; uint8_t tableType; - char sTableName[TSDB_TABLE_FNAME_LEN]; + char sTableName[TSDB_TABLE_FNAME_LEN]; //super table name, not full name } CChildTableMeta; typedef struct STableMeta { @@ -91,7 +91,7 @@ typedef struct STableMetaInfo { * 2. keep the vgroup index for multi-vnode insertion */ int32_t vgroupIndex; - char name[TSDB_TABLE_FNAME_LEN]; // (super) table name + SName name; char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql SArray *tagColList; // SArray, involved tag columns } STableMetaInfo; @@ -142,15 +142,15 @@ typedef struct SCond { } SCond; typedef struct SJoinNode { - char tableId[TSDB_TABLE_FNAME_LEN]; uint64_t uid; int16_t tagColId; + SArray* tsJoin; + SArray* tagJoin; } SJoinNode; typedef struct SJoinInfo { - bool hasJoin; - SJoinNode left; - SJoinNode right; + bool hasJoin; + SJoinNode* joinTables[TSDB_MAX_JOIN_TABLE_NUM]; } SJoinInfo; typedef struct STagCond { @@ -176,7 +176,7 @@ typedef struct SParamInfo { } SParamInfo; typedef struct STableDataBlocks { - char tableName[TSDB_TABLE_FNAME_LEN]; + SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not int64_t vgId; // virtual group id @@ -198,9 +198,10 @@ typedef struct STableDataBlocks { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. uint32_t type; // query/insert type + STimeWindow window; // the whole query time window - STimeWindow window; // query time window - SInterval interval; + SInterval interval; // tumble time window + SSessionWindow sessionWindow; // session time window SSqlGroupbyExpr groupbyExpr; // group by tags info SArray * colList; // SArray @@ -223,11 +224,16 @@ typedef struct SQueryInfo { int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX int16_t resColumnId; // result column id + bool distinctTag; // distinct tag or not + int32_t round; // 0/1/.... + int32_t bufLen; + char* buf; } SQueryInfo; typedef struct { int command; uint8_t msgType; + char reserve1[3]; // fix bus error on arm32 bool autoCreated; // create table if it is not existed during retrieve table meta in mnode union { @@ -235,13 +241,15 @@ typedef struct { int32_t numOfTablesInSubmit; }; - uint32_t insertType; + uint32_t insertType; // TODO remove it int32_t clauseIndex; // index of multiple subclause query char * curSql; // current sql, resume position of sql after parsing paused int8_t parseFinished; + char reserve2[3]; // fix bus error on arm32 int16_t numOfCols; + char reserve3[2]; // fix bus error on arm32 uint32_t allocSize; char * payload; int32_t payloadLen; @@ -251,10 +259,12 @@ typedef struct { int32_t numOfParams; int8_t dataSourceType; // load data from file or not + char reserve4[3]; // fix bus error on arm32 int8_t submitSchema; // submit block is built with table schema + char reserve5[3]; // fix bus error on arm32 STagData tagData; // NOTE: pTagData->data is used as a variant length array - char **pTableNameList; // all involved tableMeta list of current insert sql statement. + SName **pTableNameList; // all involved tableMeta list of current insert sql statement. int32_t numOfTables; SHashObj *pTableBlockHashList; // data block for each table @@ -274,7 +284,7 @@ typedef struct { char * pRsp; int32_t rspType; int32_t rspLen; - uint64_t qhandle; + uint64_t qId; int64_t useconds; int64_t offset; // offset value from vnode during projection query of stable int32_t row; @@ -292,9 +302,14 @@ typedef struct { SColumnIndex* pColumnIndex; SArithmeticSupport *pArithSup; // support the arithmetic expression calculation on agg functions - struct SLocalReducer *pLocalReducer; + struct SLocalMerger *pLocalMerger; } SSqlRes; +typedef struct { + char key[512]; + void *pDnodeConn; +} SRpcObj; + typedef struct STscObj { void * signature; void * pTimer; @@ -310,25 +325,26 @@ typedef struct STscObj { int64_t hbrid; struct SSqlObj * sqlList; struct SSqlStream *streamList; - SRpcCorEpSet *tscCorMgmtEpSet; - void* pDnodeConn; + SRpcObj *pRpcObj; + SRpcCorEpSet *tscCorMgmtEpSet; pthread_mutex_t mutex; int32_t numOfObj; // number of sqlObj from this tscObj } STscObj; typedef struct SSubqueryState { - int32_t numOfRemain; // the number of remain unfinished subquery + pthread_mutex_t mutex; + int8_t *states; int32_t numOfSub; // the number of total sub-queries uint64_t numOfRetrievedRows; // total number of points in this query } SSubqueryState; typedef struct SSqlObj { void *signature; - pthread_t owner; // owner of sql object, by which it is executed + int64_t owner; // owner of sql object, by which it is executed STscObj *pTscObj; int64_t rpcRid; - void (*fp)(); - void (*fetchFp)(); + __async_cb_func_t fp; + __async_cb_func_t fetchFp; void *param; int64_t stime; uint32_t queryId; @@ -351,7 +367,7 @@ typedef struct SSqlObj { int64_t svgroupRid; int64_t squeryLock; - + int32_t retryReason; // previous error code struct SSqlObj *prev, *next; int64_t self; } SSqlObj; @@ -387,8 +403,9 @@ typedef struct SSqlStream { void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable); -int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn); -void tscInitMsgsFp(); +int tscAcquireRpc(const char *key, const char *user, const char *secret,void **pRpcObj); +void tscReleaseRpc(void *param); +void tscInitMsgsFp(); int tsParseSql(SSqlObj *pSql, bool initial); @@ -402,15 +419,14 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code); int tscProcessLocalCmd(SSqlObj *pSql); int tscCfgDynamicOptions(char *msg); -int taos_retrieve(TAOS_RES *res); -int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); -void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo); +int32_t tscTansformFuncForSTableQuery(SQueryInfo *pQueryInfo); +void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); -void tscResetSqlCmdObj(SSqlCmd *pCmd); +void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta); /** * free query result of the sql object @@ -423,6 +439,8 @@ void tscFreeSqlResult(SSqlObj *pSql); * @param pObj */ void tscFreeSqlObj(SSqlObj *pSql); +void tscFreeSubobj(SSqlObj* pSql); + void tscFreeRegisteredSqlObj(void *pSql); void tscCloseTscObj(void *pObj); @@ -431,13 +449,18 @@ void tscCloseTscObj(void *pObj); TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos); TAOS_RES* taos_query_h(TAOS* taos, const char *sqlstr, int64_t* res); +TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param); + void waitForQueryRsp(void *param, TAOS_RES *tres, int code); void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, __async_cb_func_t fp, void *param, const char *sqlstr, size_t sqlLen); -void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql); +void tscImportDataFromFile(SSqlObj *pSql); void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen); bool tscIsUpdateQuery(SSqlObj* pSql); +char* tscGetSqlStr(SSqlObj* pSql); +bool tscIsQueryWithLimit(SSqlObj* pSql); + bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); @@ -463,7 +486,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : (unsigned char*)pData; } else { - assert(bytes == tDataTypeDesc[type].nSize); + assert(bytes == tDataTypes[type].bytes); pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)&pInfo->pSqlExpr->param[1].i64; pRes->length[columnIndex] = bytes; @@ -480,7 +503,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField pRes->length[columnIndex] = realLen; } else { - assert(bytes == tDataTypeDesc[type].nSize); + assert(bytes == tDataTypes[type].bytes); pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)pData; pRes->length[columnIndex] = bytes; diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index a8829499a324605036beefad62d83eccf4d1c65b..56e155311e60dd0a30d3f6b5dce99c9d4c76ec7f 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -481,15 +481,19 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn case TSDB_DATA_TYPE_BOOL: (*env)->CallVoidMethod(env, rowobj, g_rowdataSetBooleanFp, i, (jboolean)(*((char *)row[i]) == 1)); break; + case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteFp, i, (jbyte) * ((int8_t *)row[i])); break; + case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: (*env)->CallVoidMethod(env, rowobj, g_rowdataSetShortFp, i, (jshort) * ((int16_t *)row[i])); break; + case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: (*env)->CallVoidMethod(env, rowobj, g_rowdataSetIntFp, i, (jint) * (int32_t *)row[i]); break; + case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: (*env)->CallVoidMethod(env, rowobj, g_rowdataSetLongFp, i, (jlong) * ((int64_t *)row[i])); break; diff --git a/src/client/src/taos.def b/src/client/src/taos.def index 49d7290ce7bbb9cf64bd71c8d263ad3913576a71..43cd8190614facc207bbad0885360f23c8216c8a 100644 --- a/src/client/src/taos.def +++ b/src/client/src/taos.def @@ -32,7 +32,6 @@ taos_errstr taos_errno taos_query_a taos_fetch_rows_a -taos_fetch_row_a taos_subscribe taos_consume taos_unsubscribe diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 37207858a80786d60c97397bb7d579be0e25d685..0cfcff3a9869cecb318ca293fbb9e165924a1c07 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -20,24 +20,19 @@ #include "trpc.h" #include "tscLog.h" #include "tscSubquery.h" -#include "tscLocalMerge.h" #include "tscUtil.h" #include "tsched.h" #include "tschemautil.h" #include "tsclient.h" -static void tscProcessFetchRow(SSchedMsg *pMsg); static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows); -static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, void (*fp)()); - /* * Proxy function to perform sequentially query&retrieve operation. * If sql queries upon a super table and two-stage merge procedure is not involved (when employ the projection * query), it will sequentially query&retrieve data for all vnodes */ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows); -static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows); void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* param, const char* sqlstr, size_t sqlLen) { SSqlCmd* pCmd = &pSql->cmd; @@ -79,12 +74,16 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, __async_cb_func_t fp, void* para // TODO return the correct error code to client in tscQueueAsyncError void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { + taos_query_ra(taos, sqlstr, fp, param); +} + +TAOS_RES * taos_query_ra(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *param) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { tscError("bug!!! pObj:%p", pObj); terrno = TSDB_CODE_TSC_DISCONNECTED; tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return; + return NULL; } int32_t sqlLen = (int32_t)strlen(sqlstr); @@ -92,7 +91,7 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; tscQueueAsyncError(fp, param, terrno); - return; + return NULL; } nPrintTsc("%s", sqlstr); @@ -101,12 +100,15 @@ void taos_query_a(TAOS *taos, const char *sqlstr, __async_cb_func_t fp, void *pa if (pSql == NULL) { tscError("failed to malloc sqlObj"); tscQueueAsyncError(fp, param, TSDB_CODE_TSC_OUT_OF_MEMORY); - return; + return NULL; } doAsyncQuery(pObj, pSql, fp, param, sqlstr, sqlLen); + + return pSql; } + static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { if (tres == NULL) { return; @@ -148,7 +150,7 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows) { } // actual continue retrieve function with user-specified callback function -static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, void (*fp)()) { +static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRows, __async_cb_func_t fp) { SSqlObj *pSql = (SSqlObj *)tres; if (pSql == NULL) { // error tscError("sql object is NULL"); @@ -158,8 +160,8 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if ((pRes->qhandle == 0 || numOfRows != 0) && pCmd->command < TSDB_SQL_LOCAL) { - if (pRes->qhandle == 0 && numOfRows != 0) { + if ((pRes->qId == 0 || numOfRows != 0) && pCmd->command < TSDB_SQL_LOCAL) { + if (pRes->qId == 0 && numOfRows != 0) { tscError("qhandle is NULL"); } else { pRes->code = numOfRows; @@ -191,11 +193,6 @@ static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOf tscProcessAsyncRetrieveImpl(param, tres, numOfRows, tscAsyncFetchRowsProxy); } -void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows) { - // query completed, continue to retrieve - tscProcessAsyncRetrieveImpl(param, tres, numOfRows, tscAsyncFetchSingleRowProxy); -} - void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { SSqlObj *pSql = (SSqlObj *)taosa; if (pSql == NULL || pSql->signature != pSql) { @@ -211,7 +208,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { pSql->fetchFp = fp; pSql->fp = tscAsyncFetchRowsProxy; - if (pRes->qhandle == 0) { + if (pRes->qId == 0) { tscError("qhandle is NULL"); pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; pSql->param = param; @@ -263,103 +260,6 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { } } -void taos_fetch_row_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, TAOS_ROW), void *param) { - SSqlObj *pSql = (SSqlObj *)taosa; - if (pSql == NULL || pSql->signature != pSql) { - tscError("sql object is NULL"); - tscQueueAsyncError(fp, param, TSDB_CODE_TSC_DISCONNECTED); - return; - } - - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - if (pRes->qhandle == 0) { - tscError("qhandle is NULL"); - pSql->param = param; - pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - - tscAsyncResultOnError(pSql); - return; - } - - pSql->fetchFp = fp; - pSql->param = param; - - if (pRes->row >= pRes->numOfRows) { - tscResetForNextRetrieve(pRes); - pSql->fp = tscAsyncFetchSingleRowProxy; - - if (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql); - } else { - SSchedMsg schedMsg = { 0 }; - schedMsg.fp = tscProcessFetchRow; - schedMsg.ahandle = pSql; - schedMsg.thandle = pRes->tsrow; - schedMsg.msg = NULL; - taosScheduleTask(tscQhandle, &schedMsg); - } -} - -void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) { - SSqlObj *pSql = (SSqlObj *)tres; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - if (numOfRows == 0) { - if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. - tscTryQueryNextVnode(pSql, tscAsyncQuerySingleRowForNextVnode); - } else { - /* - * 1. has reach the limitation - * 2. no remain virtual nodes to be retrieved anymore - */ - (*pSql->fetchFp)(pSql->param, pSql, NULL); - } - return; - } - - for (int i = 0; i < pCmd->numOfCols; ++i){ - SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (pSup->pSqlExpr != NULL) { -// pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pSup->pSqlExpr->resBytes * pRes->row; - } else { - //todo add - } - } - - pRes->row++; - - (*pSql->fetchFp)(pSql->param, pSql, pSql->res.tsrow); -} - -void tscProcessFetchRow(SSchedMsg *pMsg) { - SSqlObj *pSql = (SSqlObj *)pMsg->ahandle; - SSqlRes *pRes = &pSql->res; - SSqlCmd *pCmd = &pSql->cmd; - - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - - for (int i = 0; i < pCmd->numOfCols; ++i) { - SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - - if (pSup->pSqlExpr != NULL) { - tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i, 0); - } else { -// todo add - } - } - - pRes->row++; - (*pSql->fetchFp)(pSql->param, pSql, pRes->tsrow); -} - // this function will be executed by queue task threads, so the terrno is not valid static void tscProcessAsyncError(SSchedMsg *pMsg) { void (*fp)() = pMsg->ahandle; @@ -372,7 +272,7 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) { int32_t* c = malloc(sizeof(int32_t)); *c = code; - SSchedMsg schedMsg = { 0 }; + SSchedMsg schedMsg = {0}; schedMsg.fp = tscProcessAsyncError; schedMsg.ahandle = fp; schedMsg.thandle = param; @@ -380,27 +280,81 @@ void tscQueueAsyncError(void(*fp), void *param, int32_t code) { taosScheduleTask(tscQhandle, &schedMsg); } - -void tscAsyncResultOnError(SSqlObj *pSql) { +static void tscAsyncResultCallback(SSchedMsg *pMsg) { + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pMsg->ahandle); if (pSql == NULL || pSql->signature != pSql) { tscDebug("%p SqlObj is freed, not add into queue async res", pSql); return; } assert(pSql->res.code != TSDB_CODE_SUCCESS); - tscError("%p invoke user specified function due to error occured, code:%s", pSql, tstrerror(pSql->res.code)); + tscError("%p invoke user specified function due to error occurred, code:%s", pSql, tstrerror(pSql->res.code)); SSqlRes *pRes = &pSql->res; if (pSql->fp == NULL || pSql->fetchFp == NULL){ + taosReleaseRef(tscObjRef, pSql->self); return; } pSql->fp = pSql->fetchFp; (*pSql->fp)(pSql->param, pSql, pRes->code); + taosReleaseRef(tscObjRef, pSql->self); +} + +void tscAsyncResultOnError(SSqlObj* pSql) { + SSchedMsg schedMsg = {0}; + schedMsg.fp = tscAsyncResultCallback; + schedMsg.ahandle = (void *)pSql->self; + schedMsg.thandle = (void *)1; + schedMsg.msg = 0; + taosScheduleTask(tscQhandle, &schedMsg); } int tscSendMsgToServer(SSqlObj *pSql); +static int32_t updateMetaBeforeRetryQuery(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SQueryInfo* pQueryInfo) { + // handle the invalid table error code for super table. + // update the pExpr info, colList info, number of table columns + // TODO Re-parse this sql and issue the corresponding subquery as an alternative for this case. + if (pSql->retryReason == TSDB_CODE_TDB_INVALID_TABLE_ID) { + int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); + + SSchema *pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); + pExpr->uid = pTableMetaInfo->pTableMeta->id.uid; + + if (pExpr->colInfo.colIndex >= 0) { + int32_t index = pExpr->colInfo.colIndex; + + if ((TSDB_COL_IS_NORMAL_COL(pExpr->colInfo.flag) && index >= numOfCols) || + (TSDB_COL_IS_TAG(pExpr->colInfo.flag) && (index < numOfCols || index >= (numOfCols + numOfTags)))) { + return pSql->retryReason; + } + + if ((pSchema[pExpr->colInfo.colIndex].colId != pExpr->colInfo.colId) && + strcasecmp(pExpr->colInfo.name, pSchema[pExpr->colInfo.colIndex].name) != 0) { + return pSql->retryReason; + } + } + } + + // validate the table columns information + for (int32_t i = 0; i < taosArrayGetSize(pQueryInfo->colList); ++i) { + SColumn *pCol = taosArrayGetP(pQueryInfo->colList, i); + if (pCol->colIndex.columnIndex >= numOfCols) { + return pSql->retryReason; + } + } + } else { + // do nothing + } + + return TSDB_CODE_SUCCESS; +} + void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)param); if (pSql == NULL) return; @@ -426,16 +380,21 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY|TSDB_QUERY_TYPE_SUBQUERY|TSDB_QUERY_TYPE_TAG_FILTER_QUERY))) { tscDebug("%p update local table meta, continue to process sql and send the corresponding query", pSql); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + code = tscGetTableMeta(pSql, pTableMetaInfo); assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS || code == TSDB_CODE_SUCCESS); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { taosReleaseRef(tscObjRef, pSql->self); return; } assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); + code = updateMetaBeforeRetryQuery(pSql, pTableMetaInfo, pQueryInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } // tscProcessSql can add error into async res tscProcessSql(pSql); @@ -459,7 +418,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { if (pCmd->command == TSDB_SQL_SELECT) { tscDebug("%p redo parse sql string and proceed", pSql); pCmd->parseFinished = false; - tscResetSqlCmdObj(pCmd); + tscResetSqlCmd(pCmd, true); code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { @@ -546,10 +505,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { return; _error: - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscAsyncResultOnError(pSql); - } - + pRes->code = code; + tscAsyncResultOnError(pSql); taosReleaseRef(tscObjRef, pSql->self); } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 48380f864105c9fe77053f05d2e49e1c9992bf55..188ba29a97c244ce5c5027c459fabc303ed85c0f 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -79,7 +79,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { char* dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 0) * totalNumOfRows + pField->bytes * i; STR_WITH_MAXSIZE_TO_VARSTR(dst, pSchema[i].name, pField->bytes); - char *type = tDataTypeDesc[pSchema[i].type].aName; + char *type = tDataTypes[pSchema[i].type].name; pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); dst = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; @@ -119,7 +119,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { // type name pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 1); - char *type = tDataTypeDesc[pSchema[i].type].aName; + char *type = tDataTypes[pSchema[i].type].name; output = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 1) * totalNumOfRows + pField->bytes * i; STR_WITH_MAXSIZE_TO_VARSTR(output, type, pField->bytes); @@ -309,7 +309,7 @@ TAOS_ROW tscFetchRow(void *param) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0 || + if (pRes->qId == 0 || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { return NULL; @@ -569,10 +569,12 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch } char fullName[TSDB_TABLE_FNAME_LEN * 2] = {0}; - extractDBName(pTableMetaInfo->name, fullName); + tNameGetDbName(&pTableMetaInfo->name, fullName); + extractTableName(pMeta->sTableName, param->sTableName); snprintf(fullName + strlen(fullName), TSDB_TABLE_FNAME_LEN - strlen(fullName), ".%s", param->sTableName); - extractTableName(pTableMetaInfo->name, param->buf); + + strncpy(param->buf, tNameGetTableName(&pTableMetaInfo->name), TSDB_TABLE_NAME_LEN); param->pParentSql = pSql; param->pInterSql = pInterSql; @@ -602,6 +604,7 @@ static int32_t tscRebuildDDLForSubTable(SSqlObj *pSql, const char *tableName, ch return TSDB_CODE_TSC_ACTION_IN_PROGRESS; } + static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, char *ddl) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -619,9 +622,9 @@ static int32_t tscRebuildDDLForNormalTable(SSqlObj *pSql, const char *tableName, if (type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes/TSDB_NCHAR_SIZE; } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypeDesc[pSchema[i].type].aName, bytes); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name, tDataTypes[pSchema[i].type].name, bytes); } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[pSchema[i].type].aName); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[pSchema[i].type].name); } } sprintf(result + strlen(result) - 1, "%s", ")"); @@ -646,9 +649,9 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, if (type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes/TSDB_NCHAR_SIZE; } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName, bytes); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result),"%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes); } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[type].aName); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name); } } snprintf(result + strlen(result) - 1, TSDB_MAX_BINARY_LEN - strlen(result), "%s %s", ")", "TAGS ("); @@ -660,9 +663,9 @@ static int32_t tscRebuildDDLForSuperTable(SSqlObj *pSql, const char *tableName, if (type == TSDB_DATA_TYPE_NCHAR) { bytes = bytes/TSDB_NCHAR_SIZE; } - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypeDesc[pSchema[i].type].aName, bytes); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s(%d),", pSchema[i].name,tDataTypes[pSchema[i].type].name, bytes); } else { - snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypeDesc[type].aName); + snprintf(result + strlen(result), TSDB_MAX_BINARY_LEN - strlen(result), "%s %s,", pSchema[i].name, tDataTypes[type].name); } } sprintf(result + strlen(result) - 1, "%s", ")"); @@ -675,8 +678,7 @@ static int32_t tscProcessShowCreateTable(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); assert(pTableMetaInfo->pTableMeta != NULL); - char tableName[TSDB_TABLE_NAME_LEN] = {0}; - extractTableName(pTableMetaInfo->name, tableName); + const char* tableName = tNameGetTableName(&pTableMetaInfo->name); char *result = (char *)calloc(1, TSDB_MAX_BINARY_LEN); int32_t code = TSDB_CODE_SUCCESS; @@ -712,7 +714,9 @@ static int32_t tscProcessShowCreateDatabase(SSqlObj *pSql) { free(pInterSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - extractTableName(pTableMetaInfo->name, param->buf); + + strncpy(param->buf, tNameGetTableName(&pTableMetaInfo->name), TSDB_TABLE_NAME_LEN); + param->pParentSql = pSql; param->pInterSql = pInterSql; param->fp = tscRebuildCreateDBStatement; @@ -745,7 +749,10 @@ static int32_t tscProcessCurrentUser(SSqlObj *pSql) { static int32_t tscProcessCurrentDB(SSqlObj *pSql) { char db[TSDB_DB_NAME_LEN] = {0}; + + pthread_mutex_lock(&pSql->pTscObj->mutex); extractDBName(pSql->pTscObj->db, db); + pthread_mutex_unlock(&pSql->pTscObj->mutex); SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); @@ -885,7 +892,12 @@ int tscProcessLocalCmd(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; if (pCmd->command == TSDB_SQL_CFG_LOCAL) { - pRes->code = (uint8_t)taosCfgDynamicOptions(pCmd->payload); + if (taosCfgDynamicOptions(pCmd->payload)) { + pRes->code = TSDB_CODE_SUCCESS; + } else { + pRes->code = TSDB_CODE_COM_INVALID_CFG_MSG; + } + pRes->numOfRows = 0; } else if (pCmd->command == TSDB_SQL_DESCRIBE_TABLE) { pRes->code = (uint8_t)tscProcessDescribeTable(pSql); } else if (pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { @@ -893,7 +905,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) { * set the qhandle to be 1 in order to pass the qhandle check, and to call partial release function to * free allocated resources and remove the SqlObj from sql query linked list */ - pRes->qhandle = 0x1; + pRes->qId = 0x1; pRes->numOfRows = 0; } else if (pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE) { pRes->code = tscProcessShowCreateTable(pSql); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 4aa751574c2b78c034d357c93b64edb7100c4696..8a301d1820d4fc88406c4fe5fa5fe708a3ee92d1 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -16,7 +16,7 @@ #include "tscLocalMerge.h" #include "tscSubquery.h" #include "os.h" -#include "qAst.h" +#include "texpr.h" #include "tlosertree.h" #include "tscLog.h" #include "tscUtil.h" @@ -56,7 +56,7 @@ int32_t treeComparator(const void *pLeft, const void *pRight, void *param) { } } -static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDescriptor *pDesc) { +static void tscInitSqlContext(SSqlCmd *pCmd, SLocalMerger *pReducer, tOrderDescriptor *pDesc) { /* * the fields and offset attributes in pCmd and pModel may be different due to * merge requirement. So, the final result in pRes structure is formatted in accordance with the pCmd object. @@ -68,7 +68,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc SQLFunctionCtx *pCtx = &pReducer->pCtx[i]; SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i); - pCtx->aOutputBuf = pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity; + pCtx->pOutput = pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity; pCtx->order = pQueryInfo->order.order; pCtx->functionId = pExpr->functionId; @@ -76,7 +76,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc int16_t offset = getColumnModelOffset(pDesc->pColumnModel, i); SSchema *pSchema = getColumnModelSchema(pDesc->pColumnModel, i); - pCtx->aInputElemBuf = pReducer->pTempBuffer->data + offset; + pCtx->pInput = pReducer->pTempBuffer->data + offset; // input data format comes from pModel pCtx->inputType = pSchema->type; @@ -86,7 +86,6 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->outputBytes = pExpr->resBytes; pCtx->outputType = pExpr->resType; - pCtx->startOffset = 0; pCtx->size = 1; pCtx->hasNull = true; pCtx->currentStage = MERGE_STAGE; @@ -94,13 +93,17 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc // for top/bottom function, the output of timestamp is the first column int32_t functionId = pExpr->functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pReducer->pCtx[0].aOutputBuf; + pCtx->ptsOutputBuf = pReducer->pCtx[0].pOutput; pCtx->param[2].i64 = pQueryInfo->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[1].i64 = pQueryInfo->order.orderColId; } else if (functionId == TSDB_FUNC_APERCT) { pCtx->param[0].i64 = pExpr->param[0].i64; pCtx->param[0].nType = pExpr->param[0].nType; + } else if (functionId == TSDB_FUNC_BLKINFO) { + pCtx->param[0].i64 = pExpr->param[0].i64; + pCtx->param[0].nType = pExpr->param[0].nType; + pCtx->numOfParams = 1; } pCtx->interBufBytes = pExpr->interBytes; @@ -118,7 +121,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc if (pExpr->functionId == TSDB_FUNC_TAG_DUMMY || pExpr->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pExpr->resBytes; pTagCtx[n++] = &pReducer->pCtx[i]; - } else if ((aAggs[pExpr->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + } else if ((aAggs[pExpr->functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { pCtx = &pReducer->pCtx[i]; } } @@ -166,7 +169,7 @@ static SFillColInfo* createFillColInfo(SQueryInfo* pQueryInfo) { return pFillCol; } -void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, +void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrderDescriptor *pDesc, SColumnModel *finalmodel, SColumnModel *pFFModel, SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; @@ -212,9 +215,9 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd return; } - size_t size = sizeof(SLocalReducer) + POINTER_BYTES * numOfFlush; + size_t size = sizeof(SLocalMerger) + POINTER_BYTES * numOfFlush; - SLocalReducer *pReducer = (SLocalReducer *) calloc(1, size); + SLocalMerger *pReducer = (SLocalMerger *) calloc(1, size); if (pReducer == NULL) { tscError("%p failed to create local merge structure, out of memory", pSql); @@ -311,7 +314,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->pCtx = (SQLFunctionCtx *)calloc(tscSqlExprNumOfExprs(pQueryInfo), sizeof(SQLFunctionCtx)); pReducer->rowSize = pMemBuffer[0]->nElemSize; - tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscRestoreFuncForSTableQuery(pQueryInfo); tscFieldInfoUpdateOffset(pQueryInfo); if (pReducer->rowSize > pMemBuffer[0]->pageSize) { @@ -372,7 +375,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->offset = (int32_t)pQueryInfo->limit.offset; - pRes->pLocalReducer = pReducer; + pRes->pLocalMerger = pReducer; pRes->numOfGroups = 0; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); @@ -383,7 +386,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (pQueryInfo->fillType != TSDB_FILL_NONE) { SFillColInfo* pFillCol = createFillColInfo(pQueryInfo); - pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, + pReducer->pFillInfo = taosCreateFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, 4096, (int32_t)pQueryInfo->fieldsInfo.numOfOutput, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit, tinfo.precision, pQueryInfo->fillType, pFillCol, pSql); } @@ -477,13 +480,13 @@ int32_t saveToBuffer(tExtMemBuffer *pMemoryBuf, tOrderDescriptor *pDesc, tFilePa return 0; } -void tscDestroyLocalReducer(SSqlObj *pSql) { +void tscDestroyLocalMerger(SSqlObj *pSql) { if (pSql == NULL) { return; } SSqlRes *pRes = &(pSql->res); - if (pRes->pLocalReducer == NULL) { + if (pRes->pLocalMerger == NULL) { return; } @@ -491,14 +494,14 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // there is no more result, so we release all allocated resource - SLocalReducer *pLocalReducer = (SLocalReducer *)atomic_exchange_ptr(&pRes->pLocalReducer, NULL); - if (pLocalReducer != NULL) { - pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo); + SLocalMerger *pLocalMerge = (SLocalMerger *)atomic_exchange_ptr(&pRes->pLocalMerger, NULL); + if (pLocalMerge != NULL) { + pLocalMerge->pFillInfo = taosDestroyFillInfo(pLocalMerge->pFillInfo); - if (pLocalReducer->pCtx != NULL) { + if (pLocalMerge->pCtx != NULL) { int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; + SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[i]; tVariantDestroy(&pCtx->tag); tfree(pCtx->resultInfo); @@ -508,31 +511,31 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { } } - tfree(pLocalReducer->pCtx); + tfree(pLocalMerge->pCtx); } - tfree(pLocalReducer->prevRowOfInput); + tfree(pLocalMerge->prevRowOfInput); - tfree(pLocalReducer->pTempBuffer); - tfree(pLocalReducer->pResultBuf); + tfree(pLocalMerge->pTempBuffer); + tfree(pLocalMerge->pResultBuf); - if (pLocalReducer->pLoserTree) { - tfree(pLocalReducer->pLoserTree->param); - tfree(pLocalReducer->pLoserTree); + if (pLocalMerge->pLoserTree) { + tfree(pLocalMerge->pLoserTree->param); + tfree(pLocalMerge->pLoserTree); } - tfree(pLocalReducer->pFinalRes); - tfree(pLocalReducer->discardData); + tfree(pLocalMerge->pFinalRes); + tfree(pLocalMerge->discardData); - tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel, pLocalReducer->finalModel, - pLocalReducer->numOfVnode); - for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) { - tfree(pLocalReducer->pLocalDataSrc[i]); + tscLocalReducerEnvDestroy(pLocalMerge->pExtMemBuffer, pLocalMerge->pDesc, pLocalMerge->resColModel, pLocalMerge->finalModel, + pLocalMerge->numOfVnode); + for (int32_t i = 0; i < pLocalMerge->numOfBuffer; ++i) { + tfree(pLocalMerge->pLocalDataSrc[i]); } - pLocalReducer->numOfBuffer = 0; - pLocalReducer->numOfCompleted = 0; - free(pLocalReducer); + pLocalMerge->numOfBuffer = 0; + pLocalMerge->numOfCompleted = 0; + free(pLocalMerge); } else { tscDebug("%p already freed or another free function is invoked", pSql); } @@ -604,7 +607,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } } -bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage *tmpBuffer) { +bool isSameGroup(SSqlCmd *pCmd, SLocalMerger *pReducer, char *pPrev, tFilePage *tmpBuffer) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // disable merge procedure for column projection query @@ -720,7 +723,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr SSchema p1 = {0}; if (pExpr->colInfo.colIndex == TSDB_TBNAME_COLUMN_INDEX) { - p1 = tGetTableNameColumnSchema(); + p1 = *tGetTbnameColumnSchema(); } else if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { p1.bytes = pExpr->resBytes; p1.type = (uint8_t) pExpr->resType; @@ -744,6 +747,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr functionId = TSDB_FUNC_FIRST; } else if (functionId == TSDB_FUNC_LAST_DST) { functionId = TSDB_FUNC_LAST; + } else if (functionId == TSDB_FUNC_STDDEV_DST) { + functionId = TSDB_FUNC_STDDEV; } int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false); @@ -795,12 +800,12 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe /** * - * @param pLocalReducer + * @param pLocalMerge * @param pOneInterDataSrc * @param treeList * @return the number of remain input source. if ret == 0, all data has been handled */ -int32_t loadNewDataFromDiskFor(SLocalReducer *pLocalReducer, SLocalDataSource *pOneInterDataSrc, +int32_t loadNewDataFromDiskFor(SLocalMerger *pLocalMerge, SLocalDataSource *pOneInterDataSrc, bool *needAdjustLoserTree) { pOneInterDataSrc->rowIdx = 0; pOneInterDataSrc->pageId += 1; @@ -817,17 +822,17 @@ int32_t loadNewDataFromDiskFor(SLocalReducer *pLocalReducer, SLocalDataSource *p #endif *needAdjustLoserTree = true; } else { - pLocalReducer->numOfCompleted += 1; + pLocalMerge->numOfCompleted += 1; pOneInterDataSrc->rowIdx = -1; pOneInterDataSrc->pageId = -1; *needAdjustLoserTree = true; } - return pLocalReducer->numOfBuffer; + return pLocalMerge->numOfBuffer; } -void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource *pOneInterDataSrc, +void adjustLoserTreeFromNewData(SLocalMerger *pLocalMerge, SLocalDataSource *pOneInterDataSrc, SLoserTreeInfo *pTree) { /* * load a new data page into memory for intermediate dataset source, @@ -835,7 +840,7 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource * */ bool needToAdjust = true; if (pOneInterDataSrc->filePage.num <= pOneInterDataSrc->rowIdx) { - loadNewDataFromDiskFor(pLocalReducer, pOneInterDataSrc, &needToAdjust); + loadNewDataFromDiskFor(pLocalMerge, pOneInterDataSrc, &needToAdjust); } /* @@ -843,7 +848,7 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource * * if the loser tree is rebuild completed, we do not need to adjust */ if (needToAdjust) { - int32_t leafNodeIdx = pTree->pNode[0].index + pLocalReducer->numOfBuffer; + int32_t leafNodeIdx = pTree->pNode[0].index + pLocalMerge->numOfBuffer; #ifdef _DEBUG_VIEW printf("before adjust:\t"); @@ -860,7 +865,7 @@ void adjustLoserTreeFromNewData(SLocalReducer *pLocalReducer, SLocalDataSource * } } -void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQueryInfo, SFillInfo *pFillInfo) { +void savePrevRecordAndSetupFillInfo(SLocalMerger *pLocalMerge, SQueryInfo *pQueryInfo, SFillInfo *pFillInfo) { // discard following dataset in the same group and reset the interpolation information STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -873,28 +878,28 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ taosResetFillInfo(pFillInfo, revisedSTime); } - pLocalReducer->discard = true; - pLocalReducer->discardData->num = 0; + pLocalMerge->discard = true; + pLocalMerge->discardData->num = 0; - SColumnModel *pModel = pLocalReducer->pDesc->pColumnModel; - tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); + SColumnModel *pModel = pLocalMerge->pDesc->pColumnModel; + tColModelAppend(pModel, pLocalMerge->discardData, pLocalMerge->prevRowOfInput, 0, 1, 1); } -static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) { +static void genFinalResWithoutFill(SSqlRes* pRes, SLocalMerger *pLocalMerge, SQueryInfo* pQueryInfo) { assert(pQueryInfo->interval.interval == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); - tFilePage * pBeforeFillData = pLocalReducer->pResultBuf; + tFilePage * pBeforeFillData = pLocalMerge->pResultBuf; - pRes->data = pLocalReducer->pFinalRes; + pRes->data = pLocalMerge->pFinalRes; pRes->numOfRows = (int32_t) pBeforeFillData->num; if (pQueryInfo->limit.offset > 0) { if (pQueryInfo->limit.offset < pRes->numOfRows) { int32_t prevSize = (int32_t) pBeforeFillData->num; - tColModelErase(pLocalReducer->finalModel, pBeforeFillData, prevSize, 0, (int32_t)pQueryInfo->limit.offset - 1); + tColModelErase(pLocalMerge->finalModel, pBeforeFillData, prevSize, 0, (int32_t)pQueryInfo->limit.offset - 1); /* remove the hole in column model */ - tColModelCompact(pLocalReducer->finalModel, pBeforeFillData, prevSize); + tColModelCompact(pLocalMerge->finalModel, pBeforeFillData, prevSize); pRes->numOfRows -= (int32_t) pQueryInfo->limit.offset; pQueryInfo->limit.offset = 0; @@ -907,7 +912,7 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) { pRes->numOfRows = 0; pBeforeFillData->num = 0; - pLocalReducer->discard = true; + pLocalMerge->discard = true; return; } @@ -923,41 +928,41 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, pRes->numOfRows -= overflow; pBeforeFillData->num -= overflow; - tColModelCompact(pLocalReducer->finalModel, pBeforeFillData, prevSize); + tColModelCompact(pLocalMerge->finalModel, pBeforeFillData, prevSize); // set remain data to be discarded, and reset the interpolation information - savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo); + savePrevRecordAndSetupFillInfo(pLocalMerge, pQueryInfo, pLocalMerge->pFillInfo); } - memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalModel->rowSize)); + memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalMerge->finalModel->rowSize)); pRes->numOfClauseTotal += pRes->numOfRows; pBeforeFillData->num = 0; } /* - * Note: pRes->pLocalReducer may be null, due to the fact that "tscDestroyLocalReducer" is called + * Note: pRes->pLocalMerge may be null, due to the fact that "tscDestroyLocalMerger" is called * by "interuptHandler" function in shell */ -static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneOutput) { +static void doFillResult(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool doneOutput) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - tFilePage *pBeforeFillData = pLocalReducer->pResultBuf; + tFilePage *pBeforeFillData = pLocalMerge->pResultBuf; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SFillInfo *pFillInfo = pLocalReducer->pFillInfo; + SFillInfo *pFillInfo = pLocalMerge->pFillInfo; // todo extract function int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; - tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); + void** pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity); + pResPages[i] = calloc(1, pField->bytes * pLocalMerge->resColModel->capacity); } while (1) { - int64_t newRows = taosFillResultDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity); + int64_t newRows = taosFillResultDataBlock(pFillInfo, pResPages, pLocalMerge->resColModel->capacity); if (pQueryInfo->limit.offset < newRows) { newRows -= pQueryInfo->limit.offset; @@ -965,12 +970,12 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO if (pQueryInfo->limit.offset > 0) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset, + memmove(pResPages[i], ((char*)pResPages[i]) + pField->bytes * pQueryInfo->limit.offset, (size_t)(newRows * pField->bytes)); } } - pRes->data = pLocalReducer->pFinalRes; + pRes->data = pLocalMerge->pFinalRes; pRes->numOfRows = (int32_t) newRows; pQueryInfo->limit.offset = 0; @@ -979,14 +984,13 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO pQueryInfo->limit.offset -= newRows; pRes->numOfRows = 0; - int32_t rpoints = taosNumOfRemainRows(pFillInfo); - if (rpoints <= 0) { + if (!taosFillHasMoreResults(pFillInfo)) { if (!doneOutput) { // reduce procedure has not completed yet, but current results for fill are exhausted break; } // all output in current group are completed - int32_t totalRemainRows = (int32_t)getNumOfResWithFill(pFillInfo, actualETime, pLocalReducer->resColModel->capacity); + int32_t totalRemainRows = (int32_t)getNumOfResultsAfterFillGap(pFillInfo, actualETime, pLocalMerge->resColModel->capacity); if (totalRemainRows <= 0) { break; } @@ -1004,13 +1008,13 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO assert(pRes->numOfRows >= 0); /* set remain data to be discarded, and reset the interpolation information */ - savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo); + savePrevRecordAndSetupFillInfo(pLocalMerge, pQueryInfo, pFillInfo); } int32_t offset = 0; for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i], (size_t)(pField->bytes * pRes->numOfRows)); offset += pField->bytes; } @@ -1026,8 +1030,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO tfree(pResPages); } -static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { - SColumnModel *pColumnModel = pLocalReducer->pDesc->pColumnModel; +static void savePreviousRow(SLocalMerger *pLocalMerge, tFilePage *tmpBuffer) { + SColumnModel *pColumnModel = pLocalMerge->pDesc->pColumnModel; assert(pColumnModel->capacity == 1 && tmpBuffer->num == 1); // copy to previous temp buffer @@ -1035,26 +1039,26 @@ static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) SSchema *pSchema = getColumnModelSchema(pColumnModel, i); int16_t offset = getColumnModelOffset(pColumnModel, i); - memcpy(pLocalReducer->prevRowOfInput + offset, tmpBuffer->data + offset, pSchema->bytes); + memcpy(pLocalMerge->prevRowOfInput + offset, tmpBuffer->data + offset, pSchema->bytes); } tmpBuffer->num = 0; - pLocalReducer->hasPrevRow = true; + pLocalMerge->hasPrevRow = true; } -static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, bool needInit) { +static void doExecuteFinalMerge(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, bool needInit) { // the tag columns need to be set before all functions execution SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t j = 0; j < size; ++j) { - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[j]; + SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[j]; // tags/tags_dummy function, the tag field of SQLFunctionCtx is from the input buffer int32_t functionId = pCtx->functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS_DUMMY) { tVariantDestroy(&pCtx->tag); - char* input = pCtx->aInputElemBuf; + char* input = pCtx->pInput; if (pCtx->inputType == TSDB_DATA_TYPE_BINARY || pCtx->inputType == TSDB_DATA_TYPE_NCHAR) { assert(varDataLen(input) <= pCtx->inputBytes); @@ -1062,6 +1066,7 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, } else { tVariantCreateFromBinary(&pCtx->tag, input, pCtx->inputBytes, pCtx->inputType); } + } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, j); pCtx->param[0].i64 = pExpr->param[0].i64; @@ -1075,20 +1080,20 @@ static void doExecuteSecondaryMerge(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, } for (int32_t j = 0; j < size; ++j) { - int32_t functionId = pLocalReducer->pCtx[j].functionId; + int32_t functionId = pLocalMerge->pCtx[j].functionId; if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { continue; } - aAggs[functionId].mergeFunc(&pLocalReducer->pCtx[j]); + aAggs[functionId].mergeFunc(&pLocalMerge->pCtx[j]); } } -static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { - if (pLocalReducer->hasUnprocessedRow) { - pLocalReducer->hasUnprocessedRow = false; - doExecuteSecondaryMerge(pCmd, pLocalReducer, true); - savePreviousRow(pLocalReducer, tmpBuffer); +static void handleUnprocessedRow(SSqlCmd *pCmd, SLocalMerger *pLocalMerge, tFilePage *tmpBuffer) { + if (pLocalMerge->hasUnprocessedRow) { + pLocalMerge->hasUnprocessedRow = false; + doExecuteFinalMerge(pCmd, pLocalMerge, true); + savePreviousRow(pLocalMerge, tmpBuffer); } } @@ -1102,7 +1107,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) * the number of output result is decided by main output */ int32_t functionId = pCtx[j].functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG) { continue; } @@ -1121,7 +1126,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) * filled with the same result, which is the tags, specified in group by clause * */ -static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLocalReducer *pLocalReducer) { +static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLocalMerger *pLocalMerge) { int32_t maxBufSize = 0; // find the max tags column length to prepare the buffer size_t size = tscSqlExprNumOfExprs(pQueryInfo); @@ -1136,38 +1141,38 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo char *buf = malloc((size_t)maxBufSize); for (int32_t k = 0; k < size; ++k) { - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; + SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[k]; if (pCtx->functionId != TSDB_FUNC_TAG) { continue; } int32_t inc = numOfRes - 1; // tsdb_func_tag function only produce one row of result memset(buf, 0, (size_t)maxBufSize); - memcpy(buf, pCtx->aOutputBuf, (size_t)pCtx->outputBytes); + memcpy(buf, pCtx->pOutput, (size_t)pCtx->outputBytes); for (int32_t i = 0; i < inc; ++i) { - pCtx->aOutputBuf += pCtx->outputBytes; - memcpy(pCtx->aOutputBuf, buf, (size_t)pCtx->outputBytes); + pCtx->pOutput += pCtx->outputBytes; + memcpy(pCtx->pOutput, buf, (size_t)pCtx->outputBytes); } } free(buf); } -int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { +int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge) { size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t k = 0; k < size; ++k) { - SQLFunctionCtx* pCtx = &pLocalReducer->pCtx[k]; + SQLFunctionCtx* pCtx = &pLocalMerge->pCtx[k]; aAggs[pCtx->functionId].xFinalize(pCtx); } - pLocalReducer->hasPrevRow = false; + pLocalMerge->hasPrevRow = false; - int32_t numOfRes = (int32_t)getNumOfResultLocal(pQueryInfo, pLocalReducer->pCtx); - pLocalReducer->pResultBuf->num += numOfRes; + int32_t numOfRes = (int32_t)getNumOfResultLocal(pQueryInfo, pLocalMerge->pCtx); + pLocalMerge->pResultBuf->num += numOfRes; - fillMultiRowsOfTagsVal(pQueryInfo, numOfRes, pLocalReducer); + fillMultiRowsOfTagsVal(pQueryInfo, numOfRes, pLocalMerge); return numOfRes; } @@ -1178,22 +1183,22 @@ int32_t finalizeRes(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { * results generated by simple aggregation function, we merge them all into one points * *Exception*: column projection query, required no merge procedure */ -bool needToMerge(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { +bool needToMerge(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge, tFilePage *tmpBuffer) { int32_t ret = 0; // merge all result by default - int16_t functionId = pLocalReducer->pCtx[0].functionId; + int16_t functionId = pLocalMerge->pCtx[0].functionId; // todo opt performance - if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0))) { // column projection query + if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0) && pQueryInfo->distinctTag == false)) { // column projection query ret = 1; // disable merge procedure } else { - tOrderDescriptor *pDesc = pLocalReducer->pDesc; + tOrderDescriptor *pDesc = pLocalMerge->pDesc; if (pDesc->orderInfo.numOfCols > 0) { if (pDesc->tsOrder == TSDB_ORDER_ASC) { // asc // todo refactor comparator - ret = compare_a(pLocalReducer->pDesc, 1, 0, pLocalReducer->prevRowOfInput, 1, 0, tmpBuffer->data); + ret = compare_a(pLocalMerge->pDesc, 1, 0, pLocalMerge->prevRowOfInput, 1, 0, tmpBuffer->data); } else { // desc - ret = compare_d(pLocalReducer->pDesc, 1, 0, pLocalReducer->prevRowOfInput, 1, 0, tmpBuffer->data); + ret = compare_d(pLocalMerge->pDesc, 1, 0, pLocalMerge->prevRowOfInput, 1, 0, tmpBuffer->data); } } } @@ -1231,17 +1236,17 @@ static bool saveGroupResultInfo(SSqlObj *pSql) { /** * * @param pSql - * @param pLocalReducer + * @param pLocalMerge * @param noMoreCurrentGroupRes * @return if current group is skipped, return false, and do NOT record it into pRes->numOfGroups */ -bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCurrentGroupRes) { +bool genFinalResults(SSqlObj *pSql, SLocalMerger *pLocalMerge, bool noMoreCurrentGroupRes) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - tFilePage * pResBuf = pLocalReducer->pResultBuf; - SColumnModel *pModel = pLocalReducer->resColModel; + tFilePage * pResBuf = pLocalMerge->pResultBuf; + SColumnModel *pModel = pLocalMerge->resColModel; pRes->code = TSDB_CODE_SUCCESS; @@ -1252,11 +1257,11 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur if (pQueryInfo->slimit.offset > 0) { pRes->numOfRows = 0; pQueryInfo->slimit.offset -= 1; - pLocalReducer->discard = !noMoreCurrentGroupRes; + pLocalMerge->discard = !noMoreCurrentGroupRes; - if (pLocalReducer->discard) { - SColumnModel *pInternModel = pLocalReducer->pDesc->pColumnModel; - tColModelAppend(pInternModel, pLocalReducer->discardData, pLocalReducer->pTempBuffer->data, 0, 1, 1); + if (pLocalMerge->discard) { + SColumnModel *pInternModel = pLocalMerge->pDesc->pColumnModel; + tColModelAppend(pInternModel, pLocalMerge->discardData, pLocalMerge->pTempBuffer->data, 0, 1, 1); } return false; @@ -1265,19 +1270,14 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur tColModelCompact(pModel, pResBuf, pModel->capacity); if (tscIsSecondStageQuery(pQueryInfo)) { - doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalModel->rowSize); + doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalMerge->finalModel->rowSize); } -#ifdef _DEBUG_VIEW - printf("final result before interpo:\n"); -// tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num); -#endif - // no interval query, no fill operation if (pQueryInfo->interval.interval == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { - genFinalResWithoutFill(pRes, pLocalReducer, pQueryInfo); + genFinalResWithoutFill(pRes, pLocalMerge, pQueryInfo); } else { - SFillInfo* pFillInfo = pLocalReducer->pFillInfo; + SFillInfo* pFillInfo = pLocalMerge->pFillInfo; if (pFillInfo != NULL) { TSKEY ekey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; @@ -1285,34 +1285,34 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); } - doFillResult(pSql, pLocalReducer, noMoreCurrentGroupRes); + doFillResult(pSql, pLocalMerge, noMoreCurrentGroupRes); } return true; } -void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) {// reset output buffer to the beginning +void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge) {// reset output buffer to the beginning size_t t = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < t; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - pLocalReducer->pCtx[i].aOutputBuf = pLocalReducer->pResultBuf->data + pExpr->offset * pLocalReducer->resColModel->capacity; + pLocalMerge->pCtx[i].pOutput = pLocalMerge->pResultBuf->data + pExpr->offset * pLocalMerge->resColModel->capacity; if (pExpr->functionId == TSDB_FUNC_TOP || pExpr->functionId == TSDB_FUNC_BOTTOM || pExpr->functionId == TSDB_FUNC_DIFF) { - pLocalReducer->pCtx[i].ptsOutputBuf = pLocalReducer->pCtx[0].aOutputBuf; + pLocalMerge->pCtx[i].ptsOutputBuf = pLocalMerge->pCtx[0].pOutput; } } - memset(pLocalReducer->pResultBuf, 0, pLocalReducer->nResultBufSize + sizeof(tFilePage)); + memset(pLocalMerge->pResultBuf, 0, pLocalMerge->nResultBufSize + sizeof(tFilePage)); } -static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer *pLocalReducer) { +static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalMerger *pLocalMerge) { // In handling data in other groups, we need to reset the interpolation information for a new group data pRes->numOfRows = 0; pRes->numOfRowsGroup = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pQueryInfo->limit.offset = pLocalReducer->offset; + pQueryInfo->limit.offset = pLocalMerge->offset; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -1321,12 +1321,12 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer if (pQueryInfo->fillType != TSDB_FILL_NONE) { TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); int64_t newTime = taosTimeTruncate(skey, &pQueryInfo->interval, tinfo.precision); - taosResetFillInfo(pLocalReducer->pFillInfo, newTime); + taosResetFillInfo(pLocalMerge->pFillInfo, newTime); } } -static bool isAllSourcesCompleted(SLocalReducer *pLocalReducer) { - return (pLocalReducer->numOfBuffer == pLocalReducer->numOfCompleted); +static bool isAllSourcesCompleted(SLocalMerger *pLocalMerge) { + return (pLocalMerge->numOfBuffer == pLocalMerge->numOfCompleted); } static bool doBuildFilledResultForGroup(SSqlObj *pSql) { @@ -1334,19 +1334,19 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SFillInfo *pFillInfo = pLocalReducer->pFillInfo; + SLocalMerger *pLocalMerge = pRes->pLocalMerger; + SFillInfo *pFillInfo = pLocalMerge->pFillInfo; - if (pFillInfo != NULL && taosNumOfRemainRows(pFillInfo) > 0) { + if (pFillInfo != NULL && taosFillHasMoreResults(pFillInfo)) { assert(pQueryInfo->fillType != TSDB_FILL_NONE); - tFilePage *pFinalDataBuf = pLocalReducer->pResultBuf; + tFilePage *pFinalDataBuf = pLocalMerge->pResultBuf; int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1)); // the first column must be the timestamp column - int32_t rows = (int32_t) getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity); + int32_t rows = (int32_t) getNumOfResultsAfterFillGap(pFillInfo, etime, pLocalMerge->resColModel->capacity); if (rows > 0) { // do fill gap - doFillResult(pSql, pLocalReducer, false); + doFillResult(pSql, pLocalMerge, false); } return true; @@ -1359,23 +1359,23 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SFillInfo *pFillInfo = pLocalReducer->pFillInfo; + SLocalMerger *pLocalMerge = pRes->pLocalMerger; + SFillInfo *pFillInfo = pLocalMerge->pFillInfo; - bool prevGroupCompleted = (!pLocalReducer->discard) && pLocalReducer->hasUnprocessedRow; + bool prevGroupCompleted = (!pLocalMerge->discard) && pLocalMerge->hasUnprocessedRow; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL || + if ((isAllSourcesCompleted(pLocalMerge) && !pLocalMerge->hasPrevRow) || pLocalMerge->pLocalDataSrc[0] == NULL || prevGroupCompleted) { // if fillType == TSDB_FILL_NONE, return directly if (pQueryInfo->fillType != TSDB_FILL_NONE && ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey; - int32_t rows = (int32_t)getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity); + int32_t rows = (int32_t)getNumOfResultsAfterFillGap(pFillInfo, etime, pLocalMerge->resColModel->capacity); if (rows > 0) { - doFillResult(pSql, pLocalReducer, true); + doFillResult(pSql, pLocalMerge, true); } } @@ -1385,7 +1385,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { * * No results will be generated and query completed. */ - if (pRes->numOfRows > 0 || (isAllSourcesCompleted(pLocalReducer) && (!pLocalReducer->hasUnprocessedRow))) { + if (pRes->numOfRows > 0 || (isAllSourcesCompleted(pLocalMerge) && (!pLocalMerge->hasUnprocessedRow))) { return true; } @@ -1394,7 +1394,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { return true; } - resetEnvForNewResultset(pRes, pCmd, pLocalReducer); + resetEnvForNewResultset(pRes, pCmd, pLocalMerge); } return false; @@ -1404,13 +1404,13 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - SLocalReducer *pLocalReducer = pRes->pLocalReducer; + SLocalMerger *pLocalMerge = pRes->pLocalMerger; SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t k = 0; k < size; ++k) { - SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[k]; - pCtx->aOutputBuf += pCtx->outputBytes * numOfRes; + SQLFunctionCtx *pCtx = &pLocalMerge->pCtx[k]; + pCtx->pOutput += pCtx->outputBytes * numOfRes; // set the correct output timestamp column position if (pCtx->functionId == TSDB_FUNC_TOP || pCtx->functionId == TSDB_FUNC_BOTTOM) { @@ -1418,7 +1418,7 @@ static void doProcessResultInNextWindow(SSqlObj *pSql, int32_t numOfRes) { } } - doExecuteSecondaryMerge(pCmd, pLocalReducer, true); + doExecuteFinalMerge(pCmd, pLocalMerge, true); } int32_t tscDoLocalMerge(SSqlObj *pSql) { @@ -1427,14 +1427,18 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tscResetForNextRetrieve(pRes); - if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed + if (pSql->signature != pSql || pRes == NULL || pRes->pLocalMerger == NULL) { // all data has been processed + if (pRes->code == TSDB_CODE_SUCCESS) { + return pRes->code; + } + tscError("%p local merge abort due to error occurs, code:%s", pSql, tstrerror(pRes->code)); return pRes->code; } - SLocalReducer *pLocalReducer = pRes->pLocalReducer; + SLocalMerger *pLocalMerge = pRes->pLocalMerger; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - tFilePage *tmpBuffer = pLocalReducer->pTempBuffer; + tFilePage *tmpBuffer = pLocalMerge->pTempBuffer; if (doHandleLastRemainData(pSql)) { return TSDB_CODE_SUCCESS; @@ -1444,24 +1448,24 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } - SLoserTreeInfo *pTree = pLocalReducer->pLoserTree; + SLoserTreeInfo *pTree = pLocalMerge->pLoserTree; // clear buffer - handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); - SColumnModel *pModel = pLocalReducer->pDesc->pColumnModel; + handleUnprocessedRow(pCmd, pLocalMerge, tmpBuffer); + SColumnModel *pModel = pLocalMerge->pDesc->pColumnModel; while (1) { - if (isAllSourcesCompleted(pLocalReducer)) { + if (isAllSourcesCompleted(pLocalMerge)) { break; } #ifdef _DEBUG_VIEW printf("chosen data in pTree[0] = %d\n", pTree->pNode[0].index); #endif - assert((pTree->pNode[0].index < pLocalReducer->numOfBuffer) && (pTree->pNode[0].index >= 0) && tmpBuffer->num == 0); + assert((pTree->pNode[0].index < pLocalMerge->numOfBuffer) && (pTree->pNode[0].index >= 0) && tmpBuffer->num == 0); // chosen from loser tree - SLocalDataSource *pOneDataSrc = pLocalReducer->pLocalDataSrc[pTree->pNode[0].index]; + SLocalDataSource *pOneDataSrc = pLocalMerge->pLocalDataSrc[pTree->pNode[0].index]; tColModelAppend(pModel, tmpBuffer, pOneDataSrc->filePage.data, pOneDataSrc->rowIdx, 1, pOneDataSrc->pMemBuffer->pColumnModel->capacity); @@ -1474,76 +1478,76 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tColModelDisplayEx(pModel, tmpBuffer->data, tmpBuffer->num, pModel->capacity, colInfo); #endif - if (pLocalReducer->discard) { - assert(pLocalReducer->hasUnprocessedRow == false); + if (pLocalMerge->discard) { + assert(pLocalMerge->hasUnprocessedRow == false); /* current record belongs to the same group of previous record, need to discard it */ - if (isSameGroup(pCmd, pLocalReducer, pLocalReducer->discardData->data, tmpBuffer)) { + if (isSameGroup(pCmd, pLocalMerge, pLocalMerge->discardData->data, tmpBuffer)) { tmpBuffer->num = 0; pOneDataSrc->rowIdx += 1; - adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree); + adjustLoserTreeFromNewData(pLocalMerge, pOneDataSrc, pTree); // all inputs are exhausted, abort current process - if (isAllSourcesCompleted(pLocalReducer)) { + if (isAllSourcesCompleted(pLocalMerge)) { break; } // data belongs to the same group needs to be discarded continue; } else { - pLocalReducer->discard = false; - pLocalReducer->discardData->num = 0; + pLocalMerge->discard = false; + pLocalMerge->discardData->num = 0; if (saveGroupResultInfo(pSql)) { return TSDB_CODE_SUCCESS; } - resetEnvForNewResultset(pRes, pCmd, pLocalReducer); + resetEnvForNewResultset(pRes, pCmd, pLocalMerge); } } - if (pLocalReducer->hasPrevRow) { - if (needToMerge(pQueryInfo, pLocalReducer, tmpBuffer)) { + if (pLocalMerge->hasPrevRow) { + if (needToMerge(pQueryInfo, pLocalMerge, tmpBuffer)) { // belong to the group of the previous row, continue process it - doExecuteSecondaryMerge(pCmd, pLocalReducer, false); + doExecuteFinalMerge(pCmd, pLocalMerge, false); // copy to buffer - savePreviousRow(pLocalReducer, tmpBuffer); + savePreviousRow(pLocalMerge, tmpBuffer); } else { /* * current row does not belong to the group of previous row. * so the processing of previous group is completed. */ - int32_t numOfRes = finalizeRes(pQueryInfo, pLocalReducer); - bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); + int32_t numOfRes = finalizeRes(pQueryInfo, pLocalMerge); + bool sameGroup = isSameGroup(pCmd, pLocalMerge, pLocalMerge->prevRowOfInput, tmpBuffer); - tFilePage *pResBuf = pLocalReducer->pResultBuf; + tFilePage *pResBuf = pLocalMerge->pResultBuf; /* * if the previous group does NOT generate any result (pResBuf->num == 0), * continue to process results instead of return results. */ - if ((!sameGroup && pResBuf->num > 0) || (pResBuf->num == pLocalReducer->resColModel->capacity)) { + if ((!sameGroup && pResBuf->num > 0) || (pResBuf->num == pLocalMerge->resColModel->capacity)) { // does not belong to the same group - bool notSkipped = genFinalResults(pSql, pLocalReducer, !sameGroup); + bool notSkipped = genFinalResults(pSql, pLocalMerge, !sameGroup); // this row needs to discard, since it belongs to the group of previous - if (pLocalReducer->discard && sameGroup) { - pLocalReducer->hasUnprocessedRow = false; + if (pLocalMerge->discard && sameGroup) { + pLocalMerge->hasUnprocessedRow = false; tmpBuffer->num = 0; } else { // current row does not belongs to the previous group, so it is not be handled yet. - pLocalReducer->hasUnprocessedRow = true; + pLocalMerge->hasUnprocessedRow = true; } - resetOutputBuf(pQueryInfo, pLocalReducer); + resetOutputBuf(pQueryInfo, pLocalMerge); pOneDataSrc->rowIdx += 1; // here we do not check the return value - adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree); + adjustLoserTreeFromNewData(pLocalMerge, pOneDataSrc, pTree); if (pRes->numOfRows == 0) { - handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); + handleUnprocessedRow(pCmd, pLocalMerge, tmpBuffer); if (!sameGroup) { /* @@ -1554,7 +1558,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { return TSDB_CODE_SUCCESS; } - resetEnvForNewResultset(pRes, pCmd, pLocalReducer); + resetEnvForNewResultset(pRes, pCmd, pLocalMerge); } } else { /* @@ -1562,7 +1566,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * We start the process in a new round. */ if (sameGroup) { - handleUnprocessedRow(pCmd, pLocalReducer, tmpBuffer); + handleUnprocessedRow(pCmd, pLocalMerge, tmpBuffer); } } @@ -1574,24 +1578,24 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { } } else { // result buffer is not full doProcessResultInNextWindow(pSql, numOfRes); - savePreviousRow(pLocalReducer, tmpBuffer); + savePreviousRow(pLocalMerge, tmpBuffer); } } } else { - doExecuteSecondaryMerge(pCmd, pLocalReducer, true); - savePreviousRow(pLocalReducer, tmpBuffer); // copy the processed row to buffer + doExecuteFinalMerge(pCmd, pLocalMerge, true); + savePreviousRow(pLocalMerge, tmpBuffer); // copy the processed row to buffer } pOneDataSrc->rowIdx += 1; - adjustLoserTreeFromNewData(pLocalReducer, pOneDataSrc, pTree); + adjustLoserTreeFromNewData(pLocalMerge, pOneDataSrc, pTree); } - if (pLocalReducer->hasPrevRow) { - finalizeRes(pQueryInfo, pLocalReducer); + if (pLocalMerge->hasPrevRow) { + finalizeRes(pQueryInfo, pLocalMerge); } - if (pLocalReducer->pResultBuf->num) { - genFinalResults(pSql, pLocalReducer, true); + if (pLocalMerge->pResultBuf->num) { + genFinalResults(pSql, pLocalMerge, true); } return TSDB_CODE_SUCCESS; @@ -1599,26 +1603,26 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) { SSqlRes *pRes = &pObj->res; - if (pRes->pLocalReducer != NULL) { - tscDestroyLocalReducer(pObj); + if (pRes->pLocalMerger != NULL) { + tscDestroyLocalMerger(pObj); } - pRes->qhandle = 1; // hack to pass the safety check in fetch_row function + pRes->qId = 1; // hack to pass the safety check in fetch_row function pRes->numOfRows = 0; pRes->row = 0; pRes->rspType = 0; // used as a flag to denote if taos_retrieved() has been called yet - pRes->pLocalReducer = (SLocalReducer *)calloc(1, sizeof(SLocalReducer)); + pRes->pLocalMerger = (SLocalMerger *)calloc(1, sizeof(SLocalMerger)); /* * we need one additional byte space * the sprintf function needs one additional space to put '\0' at the end of string */ size_t allocSize = numOfRes * rowLen + sizeof(tFilePage) + 1; - pRes->pLocalReducer->pResultBuf = (tFilePage *)calloc(1, allocSize); + pRes->pLocalMerger->pResultBuf = (tFilePage *)calloc(1, allocSize); - pRes->pLocalReducer->pResultBuf->num = numOfRes; - pRes->data = pRes->pLocalReducer->pResultBuf->data; + pRes->pLocalMerger->pResultBuf->num = numOfRes; + pRes->data = pRes->pLocalMerger->pResultBuf->data; } int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 7151c33393577e877dd011b27df44dfc69be089d..2b962333d588117ba54f363b7da586b6f04e3d47 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -307,7 +307,8 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, return tscInvalidSQLErrMsg(msg, "illegal float data", pToken->z); } - *((float *)payload) = (float)dv; +// *((float *)payload) = (float)dv; + SET_FLOAT_VAL(payload, dv); } break; @@ -393,7 +394,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start TSKEY k = *(TSKEY *)start; - if (k == 0) { + if (k == INT64_MIN) { if (pDataBlocks->tsSource == TSDB_USE_CLI_TS) { return -1; } else if (pDataBlocks->tsSource == -1) { @@ -703,7 +704,7 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, SParsedDataColI STableDataBlocks *dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &dataBuf, NULL); + sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &dataBuf, NULL); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -813,26 +814,26 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { tscAddEmptyMetaInfo(pQueryInfo); } - STableMetaInfo *pSTableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); - code = tscSetTableFullName(pSTableMeterMetaInfo, &sToken, pSql); + STableMetaInfo *pSTableMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); + code = tscSetTableFullName(pSTableMetaInfo, &sToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } - tstrncpy(pCmd->tagData.name, pSTableMeterMetaInfo->name, sizeof(pCmd->tagData.name)); + tNameExtractFullName(&pSTableMetaInfo->name, pCmd->tagData.name); pCmd->tagData.dataLen = 0; - code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); + code = tscGetTableMeta(pSql, pSTableMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; } - if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMeterMetaInfo)) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pSTableMetaInfo)) { return tscInvalidSQLErrMsg(pCmd->payload, "create table only from super table is allowed", sToken.z); } - SSchema *pTagSchema = tscGetTableTagSchema(pSTableMeterMetaInfo->pTableMeta); - STableComInfo tinfo = tscGetTableInfo(pSTableMeterMetaInfo->pTableMeta); + SSchema *pTagSchema = tscGetTableTagSchema(pSTableMetaInfo->pTableMeta); + STableComInfo tinfo = tscGetTableInfo(pSTableMetaInfo->pTableMeta); index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); @@ -840,7 +841,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { SParsedDataColInfo spd = {0}; - uint8_t numOfTags = tscGetNumOfTags(pSTableMeterMetaInfo->pTableMeta); + uint8_t numOfTags = tscGetNumOfTags(pSTableMetaInfo->pTableMeta); spd.numOfCols = numOfTags; // if specify some tags column @@ -905,6 +906,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z); } + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + if (sToken.type != TK_LP) { + return tscInvalidSQLErrMsg(pCmd->payload, NULL, sToken.z); + } + SKVRowBuilder kvRowBuilder = {0}; if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1036,11 +1044,7 @@ static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) { } /** - * usage: insert into table1 values() () table2 values()() - * - * @param str - * @param acct - * @param db + * parse insert sql * @param pSql * @return */ @@ -1281,7 +1285,7 @@ int tsParseInsertSql(SSqlObj *pSql) { } if (taosHashGetSize(pCmd->pTableBlockHashList) > 0) { // merge according to vgId - if ((code = tscMergeTableDataBlocks(pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscMergeTableDataBlocks(pSql, true)) != TSDB_CODE_SUCCESS) { goto _clean; } } @@ -1336,15 +1340,6 @@ int tsParseSql(SSqlObj *pSql, bool initial) { } if (tscIsInsertData(pSql->sqlstr)) { - /* - * Set the fp before parse the sql string, in case of getTableMeta failed, in which - * the error handle callback function can rightfully restore the user-defined callback function (fp). - */ - if (initial && (pSql->cmd.insertType != TSDB_QUERY_TYPE_STMT_INSERT)) { - pSql->fetchFp = pSql->fp; - pSql->fp = (void(*)())tscHandleMultivnodeInsert; - } - if (initial && ((ret = tsInsertInitialCheck(pSql)) != TSDB_CODE_SUCCESS)) { return ret; } @@ -1352,10 +1347,11 @@ int tsParseSql(SSqlObj *pSql, bool initial) { // make a backup as tsParseInsertSql may modify the string char* sqlstr = strdup(pSql->sqlstr); ret = tsParseInsertSql(pSql); - if (sqlstr == NULL || pSql->parseRetry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) { + if ((sqlstr == NULL) || (pSql->parseRetry >= 1) || + (ret != TSDB_CODE_TSC_SQL_SYNTAX_ERROR && ret != TSDB_CODE_TSC_INVALID_SQL)) { free(sqlstr); } else { - tscResetSqlCmdObj(pCmd); + tscResetSqlCmd(pCmd, true); free(pSql->sqlstr); pSql->sqlstr = sqlstr; pSql->parseRetry++; @@ -1364,10 +1360,10 @@ int tsParseSql(SSqlObj *pSql, bool initial) { } } } else { - SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr); + SSqlInfo SQLInfo = qSqlParse(pSql->sqlstr); ret = tscToSQLCmd(pSql, &SQLInfo); if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) { - tscResetSqlCmdObj(pCmd); + tscResetSqlCmd(pCmd, true); pSql->parseRetry++; ret = tscToSQLCmd(pSql, &SQLInfo); } @@ -1398,7 +1394,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", NULL); } - if ((code = tscMergeTableDataBlocks(pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscMergeTableDataBlocks(pSql, true)) != TSDB_CODE_SUCCESS) { return code; } @@ -1415,39 +1411,38 @@ typedef struct SImportFileSupport { FILE *fp; } SImportFileSupport; -static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { +static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRows) { assert(param != NULL && tres != NULL); + char * tokenBuf = NULL; + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + int32_t count = 0; + int32_t maxRows = 0; + FILE * fp = NULL; + SSqlObj *pSql = tres; SSqlCmd *pCmd = &pSql->cmd; - SImportFileSupport *pSupporter = (SImportFileSupport *) param; + SImportFileSupport *pSupporter = (SImportFileSupport *)param; SSqlObj *pParentSql = pSupporter->pSql; - FILE *fp = pSupporter->fp; - - if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // handle error - assert(taos_errno(pSql) == code); - - do { - if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - assert(pSql->res.numOfRows == 0); - int32_t errc = fseek(fp, 0, SEEK_SET); - if (errc < 0) { - tscError("%p failed to seek SEEK_SET since:%s", pSql, tstrerror(errno)); - } else { - break; - } - } - - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); - - pParentSql->res.code = code; - tscAsyncResultOnError(pParentSql); - return; - } while (0); + fp = pSupporter->fp; + + int32_t code = pSql->res.code; + + // retry parse data from file and import data from the begining again + if (code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + assert(pSql->res.numOfRows == 0); + int32_t ret = fseek(fp, 0, SEEK_SET); + if (ret < 0) { + tscError("%p failed to seek SEEK_SET since:%s", pSql, tstrerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); + goto _error; + } + } else if (code != TSDB_CODE_SUCCESS) { + goto _error; } // accumulate the total submit records @@ -1461,28 +1456,32 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { SParsedDataColInfo spd = {.numOfCols = tinfo.numOfColumns}; tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns); - size_t n = 0; - ssize_t readLen = 0; - char * line = NULL; - int32_t count = 0; - int32_t maxRows = 0; - tfree(pCmd->pTableNameList); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); if (pCmd->pTableBlockHashList == NULL) { pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + if (pCmd->pTableBlockHashList == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } } STableDataBlocks *pTableDataBlock = NULL; - int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL); + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pTableDataBlock, NULL); if (ret != TSDB_CODE_SUCCESS) { -// return ret; + pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; } tscAllocateMemIfNeed(pTableDataBlock, tinfo.rowSize, &maxRows); - char *tokenBuf = calloc(1, 4096); + tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); + if (tokenBuf == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { @@ -1510,30 +1509,42 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { } tfree(tokenBuf); - free(line); + tfree(line); + + pParentSql->res.code = code; + if (code == TSDB_CODE_SUCCESS) { + if (count > 0) { + code = doPackSendDataBlock(pSql, count, pTableDataBlock); + if (code == TSDB_CODE_SUCCESS) { + return; + } else { + goto _error; + } + } else { + taos_free_result(pSql); + tfree(pSupporter); + fclose(fp); - if (count > 0) { - code = doPackSendDataBlock(pSql, count, pTableDataBlock); - if (code != TSDB_CODE_SUCCESS) { - pParentSql->res.code = code; - tscAsyncResultOnError(pParentSql); + pParentSql->fp = pParentSql->fetchFp; + + // all data has been sent to vnode, call user function + int32_t v = (code != TSDB_CODE_SUCCESS) ? code : (int32_t)pParentSql->res.numOfRows; + (*pParentSql->fp)(pParentSql->param, pParentSql, v); return; } + } - } else { - taos_free_result(pSql); - tfree(pSupporter); - fclose(fp); - - pParentSql->fp = pParentSql->fetchFp; +_error: + tfree(tokenBuf); + tfree(line); + taos_free_result(pSql); + tfree(pSupporter); + fclose(fp); - // all data has been sent to vnode, call user function - int32_t v = (pParentSql->res.code != TSDB_CODE_SUCCESS) ? pParentSql->res.code : (int32_t)pParentSql->res.numOfRows; - (*pParentSql->fp)(pParentSql->param, pParentSql, v); - } + tscAsyncResultOnError(pParentSql); } -void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { +void tscImportDataFromFile(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; if (pCmd->command != TSDB_SQL_INSERT) { return; @@ -1545,19 +1556,19 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL); pCmd->count = 1; - FILE *fp = fopen(pCmd->payload, "r"); + FILE *fp = fopen(pCmd->payload, "rb"); if (fp == NULL) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); tfree(pSupporter); + taos_free_result(pNew); tscAsyncResultOnError(pSql); - return; } pSupporter->pSql = pSql; - pSupporter->fp = fp; + pSupporter->fp = fp; - parseFileSendDataBlock(pSupporter, pNew, 0); + parseFileSendDataBlock(pSupporter, pNew, TSDB_CODE_SUCCESS); } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 981b354c8a964272058397e8e4b019ca49fe7ac3..4efaf7c2b516019052018c508306e4736322a284 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -255,14 +255,13 @@ static char* normalStmtBuildSql(STscStmt* stmt) { //////////////////////////////////////////////////////////////////////////////// // functions for insertion statement preparation - static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { if (bind->is_null != NULL && *(bind->is_null)) { setNull(data + param->offset, param->type, param->bytes); return TSDB_CODE_SUCCESS; } - if (1) { + if (0) { // allow user bind param data with different type union { int8_t v1; @@ -697,61 +696,44 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { SSqlCmd* pCmd = &stmt->pSql->cmd; - int32_t alloced = 1, binded = 0; - if (pCmd->batchSize > 0) { - alloced = (pCmd->batchSize + 1) / 2; - binded = pCmd->batchSize / 2; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } - size_t size = taosArrayGetSize(pCmd->pDataBlocks); - for (int32_t i = 0; i < size; ++i) { - STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i); - uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); - uint32_t dataSize = totalDataSize / alloced; - assert(dataSize * alloced == totalDataSize); - - if (alloced == binded) { - totalDataSize += dataSize + sizeof(SSubmitBlk); - if (totalDataSize > pBlock->nAllocSize) { - const double factor = 1.5; - void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); - if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - pBlock->pData = (char*)tmp; - pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); - } - } + STableDataBlocks* pBlock = NULL; - char* data = pBlock->pData + sizeof(SSubmitBlk) + dataSize * binded; - for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { - SParamInfo* param = pBlock->params + j; - int code = doBindParam(data, param, bind + param->idx); - if (code != TSDB_CODE_SUCCESS) { - tscDebug("param %d: type mismatch or invalid", param->idx); - return code; - } - } + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + if (ret != 0) { + // todo handle error } - // actual work of all data blocks is done, update block size and numOfRows. - // note we don't do this block by block during the binding process, because - // we cannot recover if something goes wrong. - pCmd->batchSize = binded * 2 + 1; + uint32_t totalDataSize = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; + if (totalDataSize > pBlock->nAllocSize) { + const double factor = 1.5; - if (binded < alloced) { - return TSDB_CODE_SUCCESS; - } + void* tmp = realloc(pBlock->pData, (uint32_t)(totalDataSize * factor)); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } - size_t total = taosArrayGetSize(pCmd->pDataBlocks); - for (int32_t i = 0; i < total; ++i) { - STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, i); + pBlock->pData = (char*)tmp; + pBlock->nAllocSize = (uint32_t)(totalDataSize * factor); + } - uint32_t totalDataSize = pBlock->size - sizeof(SSubmitBlk); - pBlock->size += totalDataSize / alloced; + char* data = pBlock->pData + sizeof(SSubmitBlk) + pBlock->rowSize * pCmd->batchSize; + for (uint32_t j = 0; j < pBlock->numOfParams; ++j) { + SParamInfo* param = &pBlock->params[j]; - SSubmitBlk* pSubmit = (SSubmitBlk*)pBlock->pData; - pSubmit->numOfRows += pSubmit->numOfRows / alloced; + int code = doBindParam(data, param, &bind[param->idx]); + if (code != TSDB_CODE_SUCCESS) { + tscDebug("param %d: type mismatch or invalid", param->idx); + return code; + } } return TSDB_CODE_SUCCESS; @@ -759,9 +741,7 @@ static int insertStmtBindParam(STscStmt* stmt, TAOS_BIND* bind) { static int insertStmtAddBatch(STscStmt* stmt) { SSqlCmd* pCmd = &stmt->pSql->cmd; - if ((pCmd->batchSize % 2) == 1) { - ++pCmd->batchSize; - } + ++pCmd->batchSize; return TSDB_CODE_SUCCESS; } @@ -793,50 +773,66 @@ static int insertStmtExecute(STscStmt* stmt) { if (pCmd->batchSize == 0) { return TSDB_CODE_TSC_INVALID_VALUE; } - if ((pCmd->batchSize % 2) == 1) { - ++pCmd->batchSize; - } - STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); assert(pCmd->numOfClause == 1); + if (taosHashGetSize(pCmd->pTableBlockHashList) == 0) { + return TSDB_CODE_SUCCESS; + } - if (taosHashGetSize(pCmd->pTableBlockHashList) > 0) { - // merge according to vgid - int code = tscMergeTableDataBlocks(stmt->pSql); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); - STableDataBlocks *pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0); - code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // set the next sent data vnode index in data block arraylist - pTableMetaInfo->vgroupIndex = 1; - } else { - pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); } - SSqlObj *pSql = stmt->pSql; - SSqlRes *pRes = &pSql->res; - pRes->numOfRows = 0; - pRes->numOfTotal = 0; - pRes->numOfClauseTotal = 0; + STableDataBlocks* pBlock = NULL; + + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + assert(ret == 0); + pBlock->size = sizeof(SSubmitBlk) + pCmd->batchSize * pBlock->rowSize; + SSubmitBlk* pBlk = (SSubmitBlk*) pBlock->pData; + pBlk->numOfRows = pCmd->batchSize; + pBlk->dataLen = 0; + pBlk->uid = pTableMeta->id.uid; + pBlk->tid = pTableMeta->id.tid; + + int code = tscMergeTableDataBlocks(stmt->pSql, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - pRes->qhandle = 0; + STableDataBlocks* pDataBlock = taosArrayGetP(pCmd->pDataBlocks, 0); + code = tscCopyDataBlockToPayload(stmt->pSql, pDataBlock); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - pSql->cmd.insertType = 0; - pSql->fetchFp = waitForQueryRsp; - pSql->fp = (void(*)())tscHandleMultivnodeInsert; + SSqlObj* pSql = stmt->pSql; + SSqlRes* pRes = &pSql->res; + pRes->numOfRows = 0; + pRes->numOfTotal = 0; - tscDoQuery(pSql); + tscProcessSql(pSql); // wait for the callback function to post the semaphore tsem_wait(&pSql->rspSem); - return pSql->res.code; + // data block reset + pCmd->batchSize = 0; + for(int32_t i = 0; i < pCmd->numOfTables; ++i) { + if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { + tfree(pCmd->pTableNameList[i]); + } + } + + pCmd->numOfTables = 0; + tfree(pCmd->pTableNameList); + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + + return pSql->res.code; } //////////////////////////////////////////////////////////////////////////////// @@ -867,11 +863,11 @@ TAOS_STMT* taos_stmt_init(TAOS* taos) { } tsem_init(&pSql->rspSem, 0, 0); - pSql->signature = pSql; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->signature = pSql; + pSql->pTscObj = pObj; + pSql->maxRetry = TSDB_MAX_REPLICA; + pStmt->pSql = pSql; - pStmt->pSql = pSql; return pStmt; } @@ -890,7 +886,9 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { SSqlRes *pRes = &pSql->res; pSql->param = (void*) pSql; pSql->fp = waitForQueryRsp; - pSql->cmd.insertType = TSDB_QUERY_TYPE_STMT_INSERT; + pSql->fetchFp = waitForQueryRsp; + + pCmd->insertType = TSDB_QUERY_TYPE_STMT_INSERT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { tscError("%p failed to malloc payload buffer", pSql); @@ -905,7 +903,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pRes->qhandle = 0; + pRes->qId = 0; pRes->numOfRows = 1; strtolower(pSql->sqlstr, sql); @@ -956,8 +954,9 @@ int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->isInsert) { return insertStmtBindParam(pStmt, bind); + } else { + return normalStmtBindParam(pStmt, bind); } - return normalStmtBindParam(pStmt, bind); } int taos_stmt_add_batch(TAOS_STMT* stmt) { @@ -981,7 +980,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; if (pStmt->isInsert) { ret = insertStmtExecute(pStmt); - } else { + } else { // normal stmt query char* sql = normalStmtBuildSql(pStmt); if (sql == NULL) { ret = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -995,6 +994,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) { free(sql); } } + return ret; } @@ -1057,14 +1057,28 @@ int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes) { } if (pStmt->isInsert) { - SSqlObj* pSql = pStmt->pSql; - SSqlCmd *pCmd = &pSql->cmd; - STableDataBlocks* pBlock = taosArrayGetP(pCmd->pDataBlocks, 0); + SSqlCmd* pCmd = &pStmt->pSql->cmd; + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0, 0); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pCmd->pTableBlockHashList == NULL) { + pCmd->pTableBlockHashList = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); + } + + STableDataBlocks* pBlock = NULL; - assert(pCmd->numOfParams == pBlock->numOfParams); - if (idx < 0 || idx >= pBlock->numOfParams) return -1; + int32_t ret = + tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_PAYLOAD_SIZE, sizeof(SSubmitBlk), + pTableMeta->tableInfo.rowSize, &pTableMetaInfo->name, pTableMeta, &pBlock, NULL); + if (ret != 0) { + // todo handle error + } + + if (idx<0 || idx>=pBlock->numOfParams) { + tscError("param %d: out of range", idx); + abort(); + } - SParamInfo* param = pBlock->params + idx; + SParamInfo* param = &pBlock->params[idx]; if (type) *type = param->type; if (bytes) *bytes = param->bytes; diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f813ff85d99e6642827a49defe6b96f29720fc57..3b0e1b5775f13e619995ef8145ba4a09533f7a49 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -233,6 +233,7 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { // We extract the lock to tscBuildHeartBeatMsg function. + int64_t now = taosGetTimestampMs(); SSqlObj *pSql = pObj->sqlList; while (pSql) { /* @@ -247,8 +248,9 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) { tstrncpy(pQdesc->sql, pSql->sqlstr, sizeof(pQdesc->sql)); pQdesc->stime = htobe64(pSql->stime); pQdesc->queryId = htonl(pSql->queryId); - pQdesc->useconds = htobe64(pSql->res.useconds); - pQdesc->qHandle = htobe64(pSql->res.qhandle); + //pQdesc->useconds = htobe64(pSql->res.useconds); + pQdesc->useconds = htobe64(now - pSql->stime); // use local time instead of sever rsp elapsed time + pQdesc->qHandle = htobe64(pSql->res.qId); pHeartbeat->numOfQueries++; pQdesc++; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ffb2927cbdbd15023572912551dfcf3646b1e60e..5841aa0cd559759225534a4b1abe95b162ba9102 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -13,14 +13,16 @@ * along with this program. If not, see . */ +#ifndef __APPLE__ #define _BSD_SOURCE #define _XOPEN_SOURCE 500 #define _DEFAULT_SOURCE #define _GNU_SOURCE +#endif // __APPLE__ #include "os.h" #include "ttype.h" -#include "qAst.h" +#include "texpr.h" #include "taos.h" #include "taosmsg.h" #include "tcompare.h" @@ -41,7 +43,7 @@ #define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } -#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) +#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_BLOCK_DIST_COLUMN_INDEX)) #define TBNAME_LIST_SEP "," typedef struct SColumnList { // todo refactor @@ -60,7 +62,7 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(SArray* pFieldList, int32_t startIdx, const char* name); -static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken); +static char* cloneCurrentDBName(SSqlObj* pSql); static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); @@ -73,35 +75,34 @@ static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int3 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); -static int32_t convertFunctionId(int32_t optr, int16_t* functionId); static uint8_t convertOptr(SStrToken *pToken); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, SArray* pSelectList, bool isSTable, bool joinQuery, bool timeWindowQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool intervalQuery); +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd); -static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySqlNode* pQuerySqlNode); +static int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* offsetToken); +static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding); static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem); -static int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql); -static int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL); -static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema* pSchema); +static int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql); +static int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySqlNode* pQuerySQL); +static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySqlNode* pQuerySqlNode, SSchema* pSchema); static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); +static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); static int32_t validateEp(char* ep); -static int32_t validateDNodeConfig(tDCLSQL* pOptions); -static int32_t validateLocalConfig(tDCLSQL* pOptions); +static int32_t validateDNodeConfig(SMiscInfo* pOptions); +static int32_t validateLocalConfig(SMiscInfo* pOptions); static int32_t validateColumnName(char* name); static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType); @@ -109,15 +110,14 @@ static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField); static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo); -static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); -static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); +static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySqlNode* pQuerySqlNode, SSqlObj* pSql); +static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql); static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); -static int32_t optrToString(tSQLExpr* pExpr, char** exprString); static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySqlNode* pQuerySqlNode); static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate); static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); @@ -125,8 +125,13 @@ static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t column static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* pInfo); static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); -static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); -static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid); +static int32_t doValidateSqlNode(SSqlObj* pSql, SQuerySqlNode* pQuerySqlNode, int32_t index); +static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid); +static bool validateDebugFlag(int32_t v); + +static bool isTimeWindowQuery(SQueryInfo* pQueryInfo) { + return pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0; +} int16_t getNewResColId(SQueryInfo* pQueryInfo) { return pQueryInfo->resColumnId--; @@ -171,6 +176,16 @@ static uint8_t convertOptr(SStrToken *pToken) { } } +static bool validateDebugFlag(int32_t v) { + const static int validFlag[] = {131, 135, 143}; + + for (int i = 0; i < tListLen(validFlag); i++) { + if (v == validFlag[i]) { + return true; + } + } + return false; +} /* * Used during parsing query sql. Since the query sql usually small in length, error position * is not needed in the final error message. @@ -229,7 +244,7 @@ static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { } int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { - if (pInfo == NULL || pSql == NULL || pSql->signature != pSql) { + if (pInfo == NULL || pSql == NULL) { return TSDB_CODE_TSC_APP_ERROR; } @@ -239,7 +254,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t code = TSDB_CODE_SUCCESS; if (!pInfo->valid || terrno == TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { terrno = TSDB_CODE_SUCCESS; // clear the error number - return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->pzErrMsg); + return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->msg); } SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); @@ -264,36 +279,48 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_DROP_DB: { const char* msg2 = "invalid name"; const char* msg3 = "param name too long"; + const char* msg4 = "table is not super table"; - SStrToken* pzName = &pInfo->pDCLInfo->a[0]; + SStrToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } if (pInfo->type == TSDB_SQL_DROP_DB) { - assert(pInfo->pDCLInfo->nTokens == 1); - - code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pzName, NULL, NULL); + assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); + code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { - assert(pInfo->pDCLInfo->nTokens == 1); + assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tscSetTableFullName(pTableMetaInfo, pzName, pSql); if(code != TSDB_CODE_SUCCESS) { return code; } + + if (pInfo->pMiscInfo->tableType == TSDB_SUPER_TABLE) { + code = tscGetTableMeta(pSql, pTableMetaInfo); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + } + } else if (pInfo->type == TSDB_SQL_DROP_DNODE) { pzName->n = strdequote(pzName->z); - strncpy(pTableMetaInfo->name, pzName->z, pzName->n); - } else { // drop user + strncpy(pCmd->payload, pzName->z, pzName->n); + } else { // drop user/account if (pzName->n >= TSDB_USER_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - strncpy(pTableMetaInfo->name, pzName->z, pzName->n); + strncpy(pCmd->payload, pzName->z, pzName->n); } break; @@ -301,13 +328,13 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_USE_DB: { const char* msg = "invalid db name"; - SStrToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pToken, NULL, NULL); + int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } @@ -332,12 +359,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg1 = "invalid db name"; const char* msg2 = "name too long"; - SCreateDBInfo* pCreateDB = &(pInfo->pDCLInfo->dbOpt); + SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); if (tscValidateName(&pCreateDB->dbname) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname), NULL, NULL); + int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname)); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -349,15 +376,15 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { break; } - case TSDB_SQL_CREATE_DNODE: { // todo hostname + case TSDB_SQL_CREATE_DNODE: { const char* msg = "invalid host name (ip address)"; - if (pInfo->pDCLInfo->nTokens > 1) { + if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - SStrToken* pIpAddr = &pInfo->pDCLInfo->a[0]; - pIpAddr->n = strdequote(pIpAddr->z); + SStrToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); + id->n = strdequote(id->z); break; } @@ -367,8 +394,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid user/account name"; const char* msg3 = "name too long"; - SStrToken* pName = &pInfo->pDCLInfo->user.user; - SStrToken* pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken* pName = &pInfo->pMiscInfo->user.user; + SStrToken* pPwd = &pInfo->pMiscInfo->user.passwd; if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -382,7 +409,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SCreateAcctSQL* pAcctOpt = &pInfo->pDCLInfo->acctOpt; + SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; if (pAcctOpt->stat.n > 0) { if (pAcctOpt->stat.z[0] == 'r' && pAcctOpt->stat.n == 1) { } else if (pAcctOpt->stat.z[0] == 'w' && pAcctOpt->stat.n == 1) { @@ -397,10 +424,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_DESCRIBE_TABLE: { - SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg1 = "invalid table name"; const char* msg2 = "table name too long"; + SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -418,10 +445,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return tscGetTableMeta(pSql, pTableMetaInfo); } case TSDB_SQL_SHOW_CREATE_TABLE: { - SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg1 = "invalid table name"; const char* msg2 = "table name is too long"; + SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -439,11 +466,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_SHOW_CREATE_DATABASE: { const char* msg1 = "invalid database name"; - SStrToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } + if (pToken->n > TSDB_DB_NAME_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -455,29 +483,34 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg3 = "invalid dnode ep"; /* validate the ip address */ - tDCLSQL* pDCL = pInfo->pDCLInfo; + SMiscInfo* pMiscInfo = pInfo->pMiscInfo; /* validate the parameter names and options */ - if (validateDNodeConfig(pDCL) != TSDB_CODE_SUCCESS) { + if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } char* pMsg = pCmd->payload; SCfgDnodeMsg* pCfg = (SCfgDnodeMsg*)pMsg; - pDCL->a[0].n = strdequote(pDCL->a[0].z); - - strncpy(pCfg->ep, pDCL->a[0].z, pDCL->a[0].n); + + SStrToken* t0 = taosArrayGet(pMiscInfo->a, 0); + SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); + + t0->n = strdequote(t0->z); + strncpy(pCfg->ep, t0->z, t0->n); if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - strncpy(pCfg->config, pDCL->a[1].z, pDCL->a[1].n); + strncpy(pCfg->config, t1->z, t1->n); + + if (taosArrayGetSize(pMiscInfo->a) == 3) { + SStrToken* t2 = taosArrayGet(pMiscInfo->a, 2); - if (pDCL->nTokens == 3) { - pCfg->config[pDCL->a[1].n] = ' '; // add sep - strncpy(&pCfg->config[pDCL->a[1].n + 1], pDCL->a[2].z, pDCL->a[2].n); + pCfg->config[t1->n] = ' '; // add sep + strncpy(&pCfg->config[t1->n + 1], t2->z, t2->n); } break; @@ -492,7 +525,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { pCmd->command = pInfo->type; - SUserInfo* pUser = &pInfo->pDCLInfo->user; + SUserInfo* pUser = &pInfo->pMiscInfo->user; SStrToken* pName = &pUser->user; SStrToken* pPwd = &pUser->passwd; @@ -536,25 +569,29 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_CFG_LOCAL: { - tDCLSQL* pDCL = pInfo->pDCLInfo; - const char* msg = "invalid configure options or values"; + SMiscInfo *pMiscInfo = pInfo->pMiscInfo; + const char *msg = "invalid configure options or values"; // validate the parameter names and options - if (validateLocalConfig(pDCL) != TSDB_CODE_SUCCESS) { + if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - strncpy(pCmd->payload, pDCL->a[0].z, pDCL->a[0].n); - if (pDCL->nTokens == 2) { - pCmd->payload[pDCL->a[0].n] = ' '; // add sep - strncpy(&pCmd->payload[pDCL->a[0].n + 1], pDCL->a[1].z, pDCL->a[1].n); - } + int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a); + assert(numOfToken >= 1 && numOfToken <= 2); - break; + SStrToken* t = taosArrayGet(pMiscInfo->a, 0); + strncpy(pCmd->payload, t->z, t->n); + if (numOfToken == 2) { + SStrToken* t1 = taosArrayGet(pMiscInfo->a, 1); + pCmd->payload[t->n] = ' '; // add sep + strncpy(&pCmd->payload[t->n + 1], t1->z, t1->n); + } + return TSDB_CODE_SUCCESS; } case TSDB_SQL_CREATE_TABLE: { - SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo; + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) { if ((code = doCheckForCreateTable(pSql, 0, pInfo)) != TSDB_CODE_SUCCESS) { @@ -589,9 +626,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { assert(pCmd->numOfClause == pInfo->subclauseInfo.numOfClause); for (int32_t i = pCmd->clauseIndex; i < pInfo->subclauseInfo.numOfClause; ++i) { - SQuerySQL* pQuerySql = pInfo->subclauseInfo.pClause[i]; + SQuerySqlNode* pQuerySqlNode = pInfo->subclauseInfo.pClause[i]; tscTrace("%p start to parse %dth subclause, total:%d", pSql, i, pInfo->subclauseInfo.numOfClause); - if ((code = doCheckForQuery(pSql, pQuerySql, i)) != TSDB_CODE_SUCCESS) { + if ((code = doValidateSqlNode(pSql, pQuerySqlNode, i)) != TSDB_CODE_SUCCESS) { return code; } @@ -633,7 +670,18 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if ((code = setKillInfo(pSql, pInfo, pInfo->type)) != TSDB_CODE_SUCCESS) { return code; } + break; + } + case TSDB_SQL_SYNC_DB_REPLICA: { + const char* msg1 = "invalid db name"; + SStrToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); + + assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); + code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); + if (code != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } break; } @@ -642,7 +690,11 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } pSql->cmd.parseFinished = 1; - return tscBuildMsg[pCmd->command](pSql, pInfo); + if (tscBuildMsg[pCmd->command] != NULL) { + return tscBuildMsg[pCmd->command](pSql, pInfo); + } else { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support sql expression"); + } } /* @@ -663,21 +715,86 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { return false; } -int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +// need to add timestamp column in result set, if it is a time window query +static int32_t addPrimaryTsColumnForTimeWindowQuery(SQueryInfo* pQueryInfo) { + uint64_t uid = tscSqlExprGet(pQueryInfo, 0)->uid; + + int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (pTableMetaInfo->pTableMeta->id.uid == uid) { + tableIndex = i; + break; + } + } + + if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tstrncpy(s.name, aAggs[TSDB_FUNC_TS].name, sizeof(s.name)); + + SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL); + return TSDB_CODE_SUCCESS; +} + +static int32_t checkInvalidExprForTimeWindow(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { const char* msg1 = "invalid query expression"; + const char* msg2 = "top/bottom query does not support order by value in time window query"; + + // for top/bottom + interval query, we do not add additional timestamp column in the front + if (isTopBottomQuery(pQueryInfo)) { + + // invalid sql: + // top(col, k) from table_name [interval(1d)|session(ts, 1d)] order by k asc + // order by normal column is not supported + int32_t colId = pQueryInfo->order.orderColId; + if (isTimeWindowQuery(pQueryInfo) && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + return TSDB_CODE_SUCCESS; + } + + /* + * invalid sql: + * select count(tbname)/count(tag1)/count(tag2) from super_table_name [interval(1d)|session(ts, 1d)]; + */ + size_t size = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < size; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + } + + /* + * invalid sql: + * select tbname, tags_fields from super_table_name [interval(1s)|session(ts,1s)] + */ + if (tscQueryTags(pQueryInfo) && isTimeWindowQuery(pQueryInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + return addPrimaryTsColumnForTimeWindowQuery(pQueryInfo); +} + +int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySqlNode* pQuerySqlNode) { const char* msg2 = "interval cannot be less than 10 ms"; const char* msg3 = "sliding cannot be used without interval"; - const char* msg4 = "top/bottom query does not support order by value in interval query"; SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0) { - if (pQuerySql->sliding.n > 0) { + if (!TPARSER_HAS_TOKEN(pQuerySqlNode->interval.interval)) { + if (TPARSER_HAS_TOKEN(pQuerySqlNode->sliding)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } + return TSDB_CODE_SUCCESS; } @@ -687,7 +804,7 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // interval is not null - SStrToken* t = &pQuerySql->interval; + SStrToken *t = &pQuerySqlNode->interval.interval; if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -704,78 +821,64 @@ int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQ } } - // for top/bottom + interval query, we do not add additional timestamp column in the front - if (isTopBottomQuery(pQueryInfo)) { - if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } + if (parseIntervalOffset(pCmd, pQueryInfo, &pQuerySqlNode->interval.offset) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } - if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } + if (parseSlidingClause(pCmd, pQueryInfo, &pQuerySqlNode->sliding) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } - int32_t colId = pQueryInfo->order.orderColId; - if (pQueryInfo->interval.interval > 0 && colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); - } + // The following part is used to check for the invalid query expression. + return checkInvalidExprForTimeWindow(pCmd, pQueryInfo); +} +int32_t parseSessionClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySqlNode * pQuerySqlNode) { + const char* msg1 = "gap should be fixed time window"; + const char* msg2 = "only one type time window allowed"; + const char* msg3 = "invalid column name"; + const char* msg4 = "invalid time window"; + + // no session window + if (!TPARSER_HAS_TOKEN(pQuerySqlNode->sessionVal.gap)) { return TSDB_CODE_SUCCESS; } - /* - * check invalid SQL: - * select count(tbname)/count(tag1)/count(tag2) from super_table_name interval(1d); - */ - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < size; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + SStrToken* col = &pQuerySqlNode->sessionVal.col; + SStrToken* gap = &pQuerySqlNode->sessionVal.gap; + + char timeUnit = 0; + if (parseNatualDuration(gap->z, gap->n, &pQueryInfo->sessionWindow.gap, &timeUnit) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - /* - * check invalid SQL: - * select tbname, tags_fields from super_table_name interval(1s) - */ - if (tscQueryTags(pQueryInfo) && pQueryInfo->interval.interval > 0) { + if (timeUnit == 'y' || timeUnit == 'n') { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - // need to add timestamp column in result set, if interval is existed - uint64_t uid = tscSqlExprGet(pQueryInfo, 0)->uid; - - int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->pTableMeta->id.uid == uid) { - tableIndex = i; - break; - } + // if the unit of time window value is millisecond, change the value from microsecond + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); + if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { + pQueryInfo->sessionWindow.gap = pQueryInfo->sessionWindow.gap / 1000; } - if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { - return TSDB_CODE_TSC_INVALID_SQL; + if (pQueryInfo->sessionWindow.gap != 0 && pQueryInfo->interval.interval != 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - SSchema s = {.bytes = TSDB_KEYSIZE, .type = TSDB_DATA_TYPE_TIMESTAMP, .colId = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tstrncpy(s.name, aAggs[TSDB_FUNC_TS].aName, sizeof(s.name)); - - SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL); - - if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + if (getColumnIndexByName(pCmd, col, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } + pQueryInfo->sessionWindow.primaryColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; - return TSDB_CODE_SUCCESS; + // The following part is used to check for the invalid query expression. + return checkInvalidExprForTimeWindow(pCmd, pQueryInfo); } -int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* offsetToken) { const char* msg1 = "interval offset cannot be negative"; const char* msg2 = "interval offset should be shorter than interval"; const char* msg3 = "cannot use 'year' as offset when interval is 'month'"; @@ -783,7 +886,7 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SStrToken* t = &pQuerySql->offset; + SStrToken* t = offsetToken; if (t->n == 0) { pQueryInfo->interval.offsetUnit = pQueryInfo->interval.intervalUnit; pQueryInfo->interval.offset = 0; @@ -826,20 +929,17 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue return TSDB_CODE_SUCCESS; } -int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding) { const char* msg0 = "sliding value too small"; const char* msg1 = "sliding value no larger than the interval value"; const char* msg2 = "sliding value can not less than 1% of interval value"; const char* msg3 = "does not support sliding when interval is natural month/year"; -// const char* msg4 = "sliding not support yet in ordinary query"; const static int32_t INTERVAL_SLIDING_FACTOR = 100; - SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SStrToken* pSliding = &pQuerySql->sliding; if (pSliding->n == 0) { pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit; pQueryInfo->interval.sliding = pQueryInfo->interval.interval; @@ -874,47 +974,60 @@ int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableName, SSqlObj* pSql) { const char* msg1 = "name too long"; + const char* msg2 = "acctId too long"; + const char* msg3 = "no acctId"; SSqlCmd* pCmd = &pSql->cmd; int32_t code = TSDB_CODE_SUCCESS; - // backup the old name in pTableMetaInfo - char oldName[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(oldName, pTableMetaInfo->name, tListLen(oldName)); + if (hasSpecifyDB(pTableName)) { // db has been specified in sql string so we ignore current db path + char* acctId = getAccountId(pSql); + if (acctId == NULL || strlen(acctId) <= 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + code = tNameSetAcctId(&pTableMetaInfo->name, acctId); + if (code != 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); - if (hasSpecifyDB(pzTableName)) { // db has been specified in sql string so we ignore current db path - code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); + code = tNameFromString(&pTableMetaInfo->name, name, T_NAME_DB|T_NAME_TABLE); if (code != 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } else { // get current DB name first, and then set it into path - SStrToken t = {0}; - getCurrentDBName(pSql, &t); - if (t.n == 0) { // current database not available or not specified - code = TSDB_CODE_TSC_DB_NOT_SELECTED; - } else { - code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); - if (code != 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); - } + char* t = cloneCurrentDBName(pSql); + if (strlen(t) == 0) { + return TSDB_CODE_TSC_DB_NOT_SELECTED; } - } - if (code != TSDB_CODE_SUCCESS) { - return code; - } + code = tNameFromString(&pTableMetaInfo->name, t, T_NAME_ACCT | T_NAME_DB); + if (code != 0) { + free(t); + return TSDB_CODE_TSC_DB_NOT_SELECTED; + } - /* - * the old name exists and is not equalled to the new name. Release the table meta - * that are corresponding to the old name for the new table name. - */ - if (strlen(oldName) > 0 && strncasecmp(oldName, pTableMetaInfo->name, tListLen(pTableMetaInfo->name)) != 0) { - tscClearTableMetaInfo(pTableMetaInfo); + free(t); + + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + + code = tNameFromString(&pTableMetaInfo->name, name, T_NAME_TABLE); + if (code != 0) { + code = invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } } - return TSDB_CODE_SUCCESS; + return code; } static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { @@ -924,7 +1037,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { const char* msg1 = "first column must be timestamp"; const char* msg2 = "row length exceeds max length"; const char* msg3 = "duplicated column names"; - const char* msg4 = "invalid data types"; + const char* msg4 = "invalid data type"; const char* msg5 = "invalid binary/nchar column length"; const char* msg6 = "invalid column name"; @@ -945,14 +1058,13 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { int32_t nLen = 0; for (int32_t i = 0; i < numOfCols; ++i) { pField = taosArrayGet(pFieldList, i); - - if (pField->bytes == 0) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + if (!isValidDataType(pField->type)) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } - if (!isValidDataType(pField->type)) { - invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + if (pField->bytes == 0) { + invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); return false; } @@ -1098,7 +1210,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { return false; } - if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_NCHAR)) { + if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); return false; } @@ -1145,7 +1257,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { const char* msg1 = "too many columns"; const char* msg2 = "duplicated column names"; const char* msg3 = "column length too long"; - const char* msg4 = "invalid data types"; + const char* msg4 = "invalid data type"; const char* msg5 = "invalid column name"; const char* msg6 = "invalid column length"; @@ -1162,7 +1274,7 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) { return false; } - if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_NCHAR) { + if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } @@ -1214,9 +1326,12 @@ static bool has(SArray* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken) { - pDBToken->z = pSql->pTscObj->db; - pDBToken->n = (uint32_t)strlen(pSql->pTscObj->db); +static char* cloneCurrentDBName(SSqlObj* pSql) { + pthread_mutex_lock(&pSql->pTscObj->mutex); + char *p = strdup(pSql->pTscObj->db); + pthread_mutex_unlock(&pSql->pTscObj->mutex); + + return p; } /* length limitation, strstr cannot be applied */ @@ -1281,7 +1396,7 @@ int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStr return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } -static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +void tscInsertPrimaryTsSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; tscColumnListInsert(pQueryInfo->colList, &tsCol); } @@ -1329,7 +1444,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo, colList, NULL); if (ret != TSDB_CODE_SUCCESS) { taosArrayDestroy(colList); - tExprTreeDestroy(&pNode, NULL); + tExprTreeDestroy(pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1338,9 +1453,9 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t for(int32_t k = 0; k < numOfNode; ++k) { SColIndex* pIndex = taosArrayGet(colList, k); if (TSDB_COL_IS_TAG(pIndex->flag)) { - tExprTreeDestroy(&pNode, NULL); + tExprTreeDestroy(pNode, NULL); taosArrayDestroy(colList); - tExprTreeDestroy(&pNode, NULL); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -1363,11 +1478,11 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); // add ts column - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); tbufCloseWriter(&bw); taosArrayDestroy(colList); - tExprTreeDestroy(&pNode, NULL); + tExprTreeDestroy(pNode, NULL); } else { columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; @@ -1399,7 +1514,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &pArithExprInfo->uid); if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); + tExprTreeDestroy(pArithExprInfo->pExpr, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); } @@ -1468,7 +1583,7 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) { // add the timestamp column into the output columns SColumnIndex index = {0}; // primary timestamp column info int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols); pSupInfo->visible = false; @@ -1476,12 +1591,26 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery) { - assert(pSelection != NULL && pCmd != NULL); +bool isValidDistinctSql(SQueryInfo* pQueryInfo) { + if (pQueryInfo == NULL) { + return false; + } + if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) != TSDB_QUERY_TYPE_STABLE_QUERY) { + return false; + } + if (tscQueryTags(pQueryInfo) && tscSqlExprNumOfExprs(pQueryInfo) == 1){ + return true; + } + return false; +} + +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, SArray* pSelectList, bool isSTable, bool joinQuery, bool timeWindowQuery) { + assert(pSelectList != NULL && pCmd != NULL); - const char* msg2 = "functions can not be mixed up"; + const char* msg2 = "functions or others can not be mixed up"; const char* msg3 = "not support query expression"; const char* msg5 = "invalid function name"; + const char* msg6 = "only support distinct one tag"; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); @@ -1489,30 +1618,34 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); } - for (int32_t i = 0; i < pSelection->nExpr; ++i) { + bool hasDistinct = false; + size_t numOfExpr = taosArrayGetSize(pSelectList); + for (int32_t i = 0; i < numOfExpr; ++i) { int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - tSqlExprItem* pItem = &pSelection->a[i]; - - // project on all fields - int32_t optr = pItem->pNode->nSQLOptr; + tSqlExprItem* pItem = taosArrayGet(pSelectList, i); + + if (hasDistinct == false) { + hasDistinct = (pItem->distinct == true); + } - if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { - // it is actually a function, but the function name is invalid - if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { + int32_t type = pItem->pNode->type; + if (type == SQL_NODE_SQLFUNCTION) { + pItem->pNode->functionId = isValidFunction(pItem->pNode->operand.z, pItem->pNode->operand.n); + if (pItem->pNode->functionId < 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 - if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - } else if (pItem->pNode->nSQLOptr >= TK_COUNT && pItem->pNode->nSQLOptr <= TK_TBID) { // sql function in selection clause, append sql function info in pSqlCmd structure sequentially if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, pItem, true) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - - } else if (pItem->pNode->nSQLOptr >= TK_PLUS && pItem->pNode->nSQLOptr <= TK_REM) { + } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { + // use the dynamic array list to decide if the function is valid or not + // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 + if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + } else if (type == SQL_NODE_EXPR) { int32_t code = handleArithmeticExpr(pCmd, clauseIndex, i, pItem); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1526,13 +1659,20 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } + if (hasDistinct == true) { + if (!isValidDistinctSql(pQueryInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + pQueryInfo->distinctTag = true; + } + // there is only one user-defined column in the final result field, add the timestamp column. size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); - if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) { addPrimaryTsColIntoResult(pQueryInfo); } - if (!functionCompatibleCheck(pQueryInfo, joinQuery, intervalQuery)) { + if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1541,7 +1681,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel * in dealing with super table queries such as: count/first/last */ if (isSTable) { - tscTansformSQLFuncForSTableQuery(pQueryInfo); + tscTansformFuncForSTableQuery(pQueryInfo); if (hasUnsupportFunctionsForSTableQuery(pCmd, pQueryInfo)) { return TSDB_CODE_TSC_INVALID_SQL; @@ -1595,7 +1735,7 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t colIndex, int32_t tabl (functionId == TSDB_FUNC_TAGPRJ)); } -SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddFuncInSelectClause(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { int16_t colId = getNewResColId(pQueryInfo); @@ -1655,7 +1795,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t const char* msg1 = "tag for normal table query is not allowed"; int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - int32_t optr = pItem->pNode->nSQLOptr; + int32_t optr = pItem->pNode->tokenId; if (optr == TK_ALL) { // project on all fields TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); @@ -1677,7 +1817,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } // add the primary timestamp column even though it is not required by user - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -1685,13 +1825,13 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t index.columnIndex = (pQueryInfo->udColumnId--); index.tableIndex = 0; - SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName); + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->value, &pItem->pNode->token, pItem->aliasName); SSqlExpr* pExpr = - tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); + tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->numOfParams = 2; - tVariantAssign(&pExpr->param[1], &pItem->pNode->val); + tVariantAssign(&pExpr->param[1], &pItem->pNode->value); } else if (optr == TK_ID) { SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -1700,8 +1840,11 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = tGetTableNameColumnSchema(); - tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG); + SSchema* colSchema = tGetTbnameColumnSchema(); + tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, colSchema, TSDB_COL_TAG); + } else if (index.columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SSchema colSchema = tGetBlockDistColumnSchema(); + tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_TAG); } else { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -1715,7 +1858,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } // add the primary timestamp column even though it is not required by user - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1738,7 +1881,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return -1; } else { type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypeDesc[type].nSize; + bytes = tDataTypes[type].bytes; } } else { type = pSchema->type; @@ -1785,9 +1928,9 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT if (tsKeepOriginalColumnName) { // keep the original column name tstrncpy(name, uname, TSDB_COL_NAME_LEN); } else { - int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1; - char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0}; - snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); + int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1; + char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].name) + 2 + 1] = {0}; + snprintf(tmp, size, "%s(%s)", aAggs[functionId].name, uname); tstrncpy(name, tmp, TSDB_COL_NAME_LEN); } @@ -1797,9 +1940,25 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT } } +static void updateLastScanOrderIfNeeded(SQueryInfo* pQueryInfo) { + if (pQueryInfo->sessionWindow.gap > 0 || tscGroupbyColumn(pQueryInfo)) { + size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExpr; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId != TSDB_FUNC_LAST && pExpr->functionId != TSDB_FUNC_LAST_DST) { + continue; + } + + pExpr->numOfParams = 1; + pExpr->param->i64 = TSDB_ORDER_ASC; + pExpr->param->nType = TSDB_DATA_TYPE_INT; + } + } +} + int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSqlExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; - int32_t optr = pItem->pNode->nSQLOptr; + int32_t functionId = pItem->pNode->functionId; const char* msg1 = "not support column types"; const char* msg2 = "invalid parameters"; @@ -1809,27 +1968,22 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col const char* msg6 = "function applied to tags not allowed"; const char* msg7 = "normal table can not apply this function"; const char* msg8 = "multi-columns selection does not support alias column name"; - const char* msg9 = "invalid function"; + const char* msg9 = "diff can no be applied to unsigned numeric type"; - switch (optr) { - case TK_COUNT: { + switch (functionId) { + case TSDB_FUNC_COUNT: { /* more than one parameter for count() function */ - if (pItem->pNode->pParam != NULL && pItem->pNode->pParam->nExpr != 1) { + if (pItem->pNode->pParam != NULL && taosArrayGetSize(pItem->pNode->pParam) != 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - int16_t functionID = 0; - if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - SSqlExpr* pExpr = NULL; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { - tSqlExprItem* pParamElem = &pItem->pNode->pParam->a[0]; + tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, 0); SStrToken* pToken = &pParamElem->pNode->colInfo; - int16_t sqlOptr = pParamElem->pNode->nSQLOptr; + int16_t sqlOptr = pParamElem->pNode->tokenId; if ((pToken->z == NULL || pToken->n == 0) && (TK_INTEGER != sqlOptr)) /*select count(1) from table*/ { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1844,20 +1998,20 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); + int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } else if (sqlOptr == TK_INTEGER) { // select count(1) from table1 char buf[8] = {0}; int64_t val = -1; - tVariant* pVariant = &pParamElem->pNode->val; + tVariant* pVariant = &pParamElem->pNode->value; if (pVariant->nType == TSDB_DATA_TYPE_BIGINT) { tVariantDump(pVariant, buf, TSDB_DATA_TYPE_BIGINT, true); val = GET_INT64_VAL(buf); } if (val == 1) { index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); + int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } else { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1876,13 +2030,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col isTag = true; } - int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, isTag); + int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, isTag); } } else { // count(*) is equalled to count(primary_timestamp_key) index = (SColumnIndex){0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - int32_t size = tDataTypeDesc[TSDB_DATA_TYPE_BIGINT].nSize; - pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); + int32_t size = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, TSDB_DATA_TYPE_BIGINT, size, getNewResColId(pQueryInfo), size, false); } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -1902,34 +2056,34 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // the time stamp may be always needed if (index.tableIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); } return TSDB_CODE_SUCCESS; } - case TK_SUM: - case TK_AVG: - case TK_RATE: - case TK_IRATE: - case TK_SUM_RATE: - case TK_SUM_IRATE: - case TK_AVG_RATE: - case TK_AVG_IRATE: - case TK_TWA: - case TK_MIN: - case TK_MAX: - case TK_DIFF: - case TK_STDDEV: - case TK_LEASTSQUARES: { + case TSDB_FUNC_SUM: + case TSDB_FUNC_AVG: + case TSDB_FUNC_RATE: + case TSDB_FUNC_IRATE: + case TSDB_FUNC_SUM_RATE: + case TSDB_FUNC_SUM_IRATE: + case TSDB_FUNC_AVG_RATE: + case TSDB_FUNC_AVG_IRATE: + case TSDB_FUNC_TWA: + case TSDB_FUNC_MIN: + case TSDB_FUNC_MAX: + case TSDB_FUNC_DIFF: + case TSDB_FUNC_STDDEV: + case TSDB_FUNC_LEASTSQR: { // 1. valid the number of parameters - if (pItem->pNode->pParam == NULL || (optr != TK_LEASTSQUARES && pItem->pNode->pParam->nExpr != 1) || - (optr == TK_LEASTSQUARES && pItem->pNode->pParam->nExpr != 3)) { + if (pItem->pNode->pParam == NULL || (functionId != TSDB_FUNC_LEASTSQR && taosArrayGetSize(pItem->pNode->pParam) != 1) || + (functionId == TSDB_FUNC_LEASTSQR && taosArrayGetSize(pItem->pNode->pParam) != 3)) { /* no parameters or more than one parameter for function */ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); - if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { + tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, 0); + if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1937,6 +2091,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if ((getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -1948,31 +2103,28 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (!IS_NUMERIC_TYPE(colType)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } else if (IS_UNSIGNED_NUMERIC_TYPE(colType) && functionId == TSDB_FUNC_DIFF) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); } int16_t resultType = 0; int16_t resultSize = 0; int32_t intermediateResSize = 0; - int16_t functionID = 0; - if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - - if (getResultDataInfo(pSchema->type, pSchema->bytes, functionID, 0, &resultType, &resultSize, + if (getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &intermediateResSize, 0, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } // set the first column ts for diff query - if (optr == TK_DIFF) { + if (functionId == TSDB_FUNC_DIFF) { colIndex += 1; SColumnIndex indexTS = {.tableIndex = index.tableIndex, .columnIndex = 0}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &indexTS, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pQueryInfo), TSDB_KEYSIZE, false); SColumnList ids = getColumnList(1, 0, 0); - insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].aName, pExpr); + insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, aAggs[TSDB_FUNC_TS_DUMMY].name, pExpr); } // functions can not be applied to tags @@ -1980,19 +2132,19 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); - if (optr == TK_LEASTSQUARES) { + if (functionId == TSDB_FUNC_LEASTSQR) { /* set the leastsquares parameters */ char val[8] = {0}; - if (tVariantDump(&pParamElem[1].pNode->val, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { + if (tVariantDump(&pParamElem[1].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { return TSDB_CODE_TSC_INVALID_SQL; } addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, DOUBLE_BYTES); memset(val, 0, tListLen(val)); - if (tVariantDump(&pParamElem[2].pNode->val, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { + if (tVariantDump(&pParamElem[2].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2015,46 +2167,41 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } } - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); return TSDB_CODE_SUCCESS; } - case TK_FIRST: - case TK_LAST: - case TK_SPREAD: - case TK_LAST_ROW: - case TK_INTERP: { + case TSDB_FUNC_FIRST: + case TSDB_FUNC_LAST: + case TSDB_FUNC_SPREAD: + case TSDB_FUNC_LAST_ROW: + case TSDB_FUNC_INTERP: { bool requireAllFields = (pItem->pNode->pParam == NULL); - int16_t functionID = 0; - if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9); - } - // NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query - SConvertFunc cvtFunc = {.originFuncId = functionID, .execFuncId = functionID}; - if (functionID == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) { + SConvertFunc cvtFunc = {.originFuncId = functionId, .execFuncId = functionId}; + if (functionId == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) { cvtFunc.execFuncId = TSDB_FUNC_LAST; } if (!requireAllFields) { - if (pItem->pNode->pParam->nExpr < 1) { + if (taosArrayGetSize(pItem->pNode->pParam) < 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (pItem->pNode->pParam->nExpr > 1 && (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0)) { + if (taosArrayGetSize(pItem->pNode->pParam) > 1 && (pItem->aliasName != NULL && strlen(pItem->aliasName) > 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); } /* in first/last function, multiple columns can be add to resultset */ - for (int32_t i = 0; i < pItem->pNode->pParam->nExpr; ++i) { - tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[i]); - if (pParamElem->pNode->nSQLOptr != TK_ALL && pParamElem->pNode->nSQLOptr != TK_ID) { + for (int32_t i = 0; i < taosArrayGetSize(pItem->pNode->pParam); ++i) { + tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, i); + if (pParamElem->pNode->tokenId != TK_ALL && pParamElem->pNode->tokenId != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* + if (pParamElem->pNode->tokenId == TK_ALL) { // select table.* SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -2088,31 +2235,14 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } char name[TSDB_COL_NAME_LEN] = {0}; - SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - bool multiColOutput = pItem->pNode->pParam->nExpr > 1; + bool multiColOutput = taosArrayGetSize(pItem->pNode->pParam) > 1; setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo, multiColOutput); - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex++, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } - - if (optr == TK_LAST) { // todo refactor - SSqlGroupbyExpr* pGroupBy = &pQueryInfo->groupbyExpr; - if (pGroupBy->numOfGroupCols > 0) { - for(int32_t k = 0; k < pGroupBy->numOfGroupCols; ++k) { - SColIndex* pIndex = taosArrayGet(pGroupBy->columnInfo, k); - if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // group by normal columns - SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, colIndex + i); - pExpr->numOfParams = 1; - pExpr->param->i64 = TSDB_ORDER_ASC; - - break; - } - } - } - } } } @@ -2139,29 +2269,28 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } - colIndex++; } numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); } - return TSDB_CODE_SUCCESS; } } - case TK_TOP: - case TK_BOTTOM: - case TK_PERCENTILE: - case TK_APERCENTILE: { + + case TSDB_FUNC_TOP: + case TSDB_FUNC_BOTTOM: + case TSDB_FUNC_PERCT: + case TSDB_FUNC_APERCT: { // 1. valid the number of parameters - if (pItem->pNode->pParam == NULL || pItem->pNode->pParam->nExpr != 2) { + if (pItem->pNode->pParam == NULL || taosArrayGetSize(pItem->pNode->pParam) != 2) { /* no parameters or more than one parameter for function */ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tSqlExprItem* pParamElem = &(pItem->pNode->pParam->a[0]); - if (pParamElem->pNode->nSQLOptr != TK_ID) { + tSqlExprItem* pParamElem = taosArrayGet(pItem->pNode->pParam, 0); + if (pParamElem->pNode->tokenId != TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -2169,6 +2298,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -2188,11 +2318,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } // 3. valid the parameters - if (pParamElem[1].pNode->nSQLOptr == TK_ID) { + if (pParamElem[1].pNode->tokenId == TK_ID) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tVariant* pVariant = &pParamElem[1].pNode->val; + tVariant* pVariant = &pParamElem[1].pNode->value; int8_t resultType = pSchema[index.columnIndex].type; int16_t resultSize = pSchema[index.columnIndex].bytes; @@ -2200,7 +2330,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col char val[8] = {0}; SSqlExpr* pExpr = NULL; - if (optr == TK_PERCENTILE || optr == TK_APERCENTILE) { + if (functionId == TSDB_FUNC_PERCT || functionId == TSDB_FUNC_APERCT) { tVariantDump(pVariant, val, TSDB_DATA_TYPE_DOUBLE, true); double dp = GET_DOUBLE_VAL(val); @@ -2216,11 +2346,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col * for dp = 0, it is actually min, * for dp = 100, it is max, */ - int16_t functionId = 0; - if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + tscInsertPrimaryTsSourceColumn(pQueryInfo, &index); colIndex += 1; // the first column is ts pExpr = tscSqlExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pQueryInfo), resultSize, false); @@ -2233,22 +2359,17 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - int16_t functionId = 0; - if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - // todo REFACTOR // set the first column ts for top/bottom query - SColumnIndex index1 = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; + SColumnIndex index1 = {index.tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS, &index1, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, getNewResColId(pQueryInfo), TSDB_KEYSIZE, false); - tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].aName, sizeof(pExpr->aliasName)); + tstrncpy(pExpr->aliasName, aAggs[TSDB_FUNC_TS].name, sizeof(pExpr->aliasName)); const int32_t TS_COLUMN_INDEX = PRIMARYKEY_TIMESTAMP_COL_INDEX; - SColumnList ids = getColumnList(1, 0, TS_COLUMN_INDEX); + SColumnList ids = getColumnList(1, index.tableIndex, TS_COLUMN_INDEX); insertResultField(pQueryInfo, TS_COLUMN_INDEX, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, - aAggs[TSDB_FUNC_TS].aName, pExpr); + aAggs[TSDB_FUNC_TS].name, pExpr); colIndex += 1; // the first column is ts @@ -2259,7 +2380,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col memset(pExpr->aliasName, 0, tListLen(pExpr->aliasName)); getColumnName(pItem, pExpr->aliasName, sizeof(pExpr->aliasName) - 1); - SColumnList ids = getColumnList(1, 0, index.columnIndex); + SColumnList ids = getColumnList(1, index.tableIndex, index.columnIndex); if (finalResult) { insertResultField(pQueryInfo, colIndex, &ids, resultSize, resultType, pExpr->aliasName, pExpr); } else { @@ -2271,19 +2392,20 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return TSDB_CODE_SUCCESS; }; - case TK_TBID: { + case TSDB_FUNC_TID_TAG: { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } // no parameters or more than one parameter for function - if (pItem->pNode->pParam == NULL || pItem->pNode->pParam->nExpr != 1) { + if (pItem->pNode->pParam == NULL || taosArrayGetSize(pItem->pNode->pParam) != 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tSQLExpr* pParam = pItem->pNode->pParam->a[0].pNode; - + tSqlExprItem* pParamItem = taosArrayGet(pItem->pNode->pParam, 0); + tSqlExpr* pParam = pParamItem->pNode; + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pParam->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -2319,7 +2441,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SSchema s = {0}; if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - s = tGetTableNameColumnSchema(); + s = *tGetTbnameColumnSchema(); } else { s = pTagSchema[index.columnIndex]; } @@ -2335,11 +2457,35 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col s.bytes = bytes; TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TID_TAG, &index, &s, TSDB_COL_TAG); return TSDB_CODE_SUCCESS; } - + case TSDB_FUNC_BLKINFO: { + // no parameters or more than one parameter for function + if (pItem->pNode->pParam != NULL && taosArrayGetSize(pItem->pNode->pParam) != 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + SColumnIndex index = {.tableIndex = 0, .columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX,}; + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + + SSchema s = {.name = "block_dist", .type = TSDB_DATA_TYPE_BINARY}; + int32_t inter = 0; + int16_t resType = 0; + int16_t bytes = 0; + getResultDataInfo(TSDB_DATA_TYPE_INT, 4, TSDB_FUNC_BLKINFO, 0, &resType, &bytes, &inter, 0, 0); + + s.bytes = bytes; + s.type = (uint8_t)resType; + SSqlExpr* pExpr = tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_BLKINFO, &index, &s, TSDB_COL_TAG); + pExpr->numOfParams = 1; + pExpr->param[0].i64 = pTableMetaInfo->pTableMeta->tableInfo.rowSize; + pExpr->param[0].nType = TSDB_DATA_TYPE_BIGINT; + + return TSDB_CODE_SUCCESS; + } + default: return TSDB_CODE_TSC_INVALID_SQL; } @@ -2364,8 +2510,8 @@ void getColumnName(tSqlExprItem* pItem, char* resultFieldName, int32_t nameLengt if (pItem->aliasName != NULL) { strncpy(resultFieldName, pItem->aliasName, nameLength); } else { - int32_t len = ((int32_t)pItem->pNode->operand.n < nameLength) ? (int32_t)pItem->pNode->operand.n : nameLength; - strncpy(resultFieldName, pItem->pNode->operand.z, len); + int32_t len = ((int32_t)pItem->pNode->token.n < nameLength) ? (int32_t)pItem->pNode->token.n : nameLength; + strncpy(resultFieldName, pItem->pNode->token.z, len); } } @@ -2377,6 +2523,14 @@ static bool isTablenameToken(SStrToken* token) { return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); } +static bool isTableBlockDistToken(SStrToken* token) { + SStrToken tmpToken = *token; + SStrToken tableToken = {0}; + + extractTableNameFromToken(&tmpToken, &tableToken); + + return (strncasecmp(TSQL_BLOCK_DIST, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_BLOCK_DIST_L)); +} static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; @@ -2406,6 +2560,8 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQu if (isTablenameToken(pToken)) { pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX; + } else if (isTableBlockDistToken(pToken)) { + pIndex->columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX; } else if (strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) { pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX; } else { @@ -2495,87 +2651,6 @@ int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* return doGetColumnIndexByName(pCmd, &tmpToken, pQueryInfo, pIndex); } -int32_t convertFunctionId(int32_t optr, int16_t* functionId) { - switch (optr) { - case TK_COUNT: - *functionId = TSDB_FUNC_COUNT; - break; - case TK_SUM: - *functionId = TSDB_FUNC_SUM; - break; - case TK_AVG: - *functionId = TSDB_FUNC_AVG; - break; - case TK_RATE: - *functionId = TSDB_FUNC_RATE; - break; - case TK_IRATE: - *functionId = TSDB_FUNC_IRATE; - break; - case TK_SUM_RATE: - *functionId = TSDB_FUNC_SUM_RATE; - break; - case TK_SUM_IRATE: - *functionId = TSDB_FUNC_SUM_IRATE; - break; - case TK_AVG_RATE: - *functionId = TSDB_FUNC_AVG_RATE; - break; - case TK_AVG_IRATE: - *functionId = TSDB_FUNC_AVG_IRATE; - break; - case TK_MIN: - *functionId = TSDB_FUNC_MIN; - break; - case TK_MAX: - *functionId = TSDB_FUNC_MAX; - break; - case TK_STDDEV: - *functionId = TSDB_FUNC_STDDEV; - break; - case TK_PERCENTILE: - *functionId = TSDB_FUNC_PERCT; - break; - case TK_APERCENTILE: - *functionId = TSDB_FUNC_APERCT; - break; - case TK_FIRST: - *functionId = TSDB_FUNC_FIRST; - break; - case TK_LAST: - *functionId = TSDB_FUNC_LAST; - break; - case TK_LEASTSQUARES: - *functionId = TSDB_FUNC_LEASTSQR; - break; - case TK_TOP: - *functionId = TSDB_FUNC_TOP; - break; - case TK_BOTTOM: - *functionId = TSDB_FUNC_BOTTOM; - break; - case TK_DIFF: - *functionId = TSDB_FUNC_DIFF; - break; - case TK_SPREAD: - *functionId = TSDB_FUNC_SPREAD; - break; - case TK_TWA: - *functionId = TSDB_FUNC_TWA; - break; - case TK_INTERP: - *functionId = TSDB_FUNC_INTERP; - break; - case TK_LAST_ROW: - *functionId = TSDB_FUNC_LAST_ROW; - break; - default: - return -1; - } - - return TSDB_CODE_SUCCESS; -} - int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); @@ -2591,10 +2666,10 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg6 = "pattern string is empty"; /* - * database prefix in pInfo->pDCLInfo->a[0] - * wildcard in like clause in pInfo->pDCLInfo->a[1] + * database prefix in pInfo->pMiscInfo->a[0] + * wildcard in like clause in pInfo->pMiscInfo->a[1] */ - SShowInfo* pShowInfo = &pInfo->pDCLInfo->showOpt; + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { // db prefix in tagCond, show table conds in payload @@ -2613,7 +2688,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - int32_t ret = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), pDbPrefixToken, NULL, NULL); + int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pDbPrefixToken); if (ret != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -2646,8 +2721,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (!validateIpAddress(pDnodeIp->z, pDnodeIp->n)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } - } - + } return TSDB_CODE_SUCCESS; } @@ -2659,7 +2733,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { SSqlCmd* pCmd = &pSql->cmd; pCmd->command = pInfo->type; - SStrToken* idStr = &(pInfo->pDCLInfo->ip); + SStrToken* idStr = &(pInfo->pMiscInfo->id); if (idStr->n > TSDB_KILL_MSG_LEN) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2704,7 +2778,7 @@ bool validateIpAddress(const char* ip, size_t size) { return epAddr != INADDR_NONE; } -int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { +int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (pTableMetaInfo->pTableMeta == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { @@ -2726,7 +2800,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { SSchema* pSrcSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, colIndex); if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || - (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) || + (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_STDDEV_DST) || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->param[0].i64, &type, &bytes, &interBytes, 0, true) != TSDB_CODE_SUCCESS) { @@ -2744,7 +2818,7 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { } /* transfer the field-info back to original input format */ -void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { +void tscRestoreFuncForSTableQuery(SQueryInfo* pQueryInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return; @@ -2768,6 +2842,8 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { functionId = TSDB_FUNC_FIRST; } else if (functionId == TSDB_FUNC_LAST_DST) { functionId = TSDB_FUNC_LAST; + } else if (functionId == TSDB_FUNC_STDDEV_DST) { + functionId = TSDB_FUNC_STDDEV; } getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &pExpr->resType, &pExpr->resBytes, @@ -2784,7 +2860,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) size_t size = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < size; ++i) { int32_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_STABLE) == 0) { + if ((aAggs[functionId].status & TSDB_FUNCSTATE_STABLE) == 0) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); return true; } @@ -2811,23 +2887,40 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool intervalQuery) { +static bool groupbyTagsOrNull(SQueryInfo* pQueryInfo) { + if (pQueryInfo->groupbyExpr.columnInfo == NULL || + taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo) == 0) { + return true; + } + + size_t s = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); + for (int32_t i = 0; i < s; i++) { + SColIndex* colIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, i); + if (colIndex->flag != TSDB_COL_TAG) { + return false; + } + } + + return true; +} + +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool twQuery) { int32_t startIdx = 0; size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo); assert(numOfExpr > 0); SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); - int32_t functionID = pExpr->functionId; // ts function can be simultaneously used with any other functions. + int32_t functionID = pExpr->functionId; if (functionID == TSDB_FUNC_TS || functionID == TSDB_FUNC_TS_DUMMY) { startIdx++; } int32_t factor = functionCompatList[tscSqlExprGet(pQueryInfo, startIdx)->functionId]; - if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_LAST_ROW && (joinQuery || intervalQuery)) { + if (tscSqlExprGet(pQueryInfo, 0)->functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) { return false; } @@ -2855,7 +2948,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery, bool } } - if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || intervalQuery)) { + if (functionId == TSDB_FUNC_LAST_ROW && (joinQuery || twQuery || !groupbyTagsOrNull(pQueryInfo))) { return false; } } @@ -2894,7 +2987,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) STableMeta* pTableMeta = NULL; SSchema* pSchema = NULL; - SSchema s = tscGetTbnameColumnSchema(); int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; @@ -2921,7 +3013,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) int32_t numOfCols = tscGetNumOfColumns(pTableMeta); if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - pSchema = &s; + pSchema = tGetTbnameColumnSchema(); } else { pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); } @@ -2996,10 +3088,10 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { } static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnFilterInfo* pColumnFilter, - SColumnIndex* columnIndex, tSQLExpr* pExpr) { + SColumnIndex* columnIndex, tSqlExpr* pExpr) { const char* msg = "not supported filter condition"; - tSQLExpr* pRight = pExpr->pRight; + tSqlExpr* pRight = pExpr->pRight; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, columnIndex->tableIndex); SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex->columnIndex); @@ -3009,39 +3101,47 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, colType = TSDB_DATA_TYPE_BIGINT; } else if (colType == TSDB_DATA_TYPE_FLOAT || colType == TSDB_DATA_TYPE_DOUBLE) { colType = TSDB_DATA_TYPE_DOUBLE; - } else if ((colType == TSDB_DATA_TYPE_TIMESTAMP) && (TSDB_DATA_TYPE_BINARY == pRight->val.nType)) { - int retVal = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, &pRight->val); + } else if ((colType == TSDB_DATA_TYPE_TIMESTAMP) && (TSDB_DATA_TYPE_BINARY == pRight->value.nType)) { + int retVal = setColumnFilterInfoForTimestamp(pCmd, pQueryInfo, &pRight->value); if (TSDB_CODE_SUCCESS != retVal) { return retVal; } } int32_t retVal = TSDB_CODE_SUCCESS; - if (pExpr->nSQLOptr == TK_LE || pExpr->nSQLOptr == TK_LT) { - retVal = tVariantDump(&pRight->val, (char*)&pColumnFilter->upperBndd, colType, false); + + int32_t bufLen = 0; + if (IS_NUMERIC_TYPE(pRight->value.nType)) { + bufLen = 60; + } else { + bufLen = pRight->value.nLen + 1; + } + + if (pExpr->tokenId == TK_LE || pExpr->tokenId == TK_LT) { + retVal = tVariantDump(&pRight->value, (char*)&pColumnFilter->upperBndd, colType, false); // TK_GT,TK_GE,TK_EQ,TK_NE are based on the pColumn->lowerBndd } else if (colType == TSDB_DATA_TYPE_BINARY) { - pColumnFilter->pz = (int64_t)calloc(1, pRight->val.nLen + TSDB_NCHAR_SIZE); - pColumnFilter->len = pRight->val.nLen; - retVal = tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType, false); + pColumnFilter->pz = (int64_t)calloc(1, bufLen * TSDB_NCHAR_SIZE); + pColumnFilter->len = pRight->value.nLen; + retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); } else if (colType == TSDB_DATA_TYPE_NCHAR) { - // pRight->val.nLen + 1 is larger than the actual nchar string length - pColumnFilter->pz = (int64_t)calloc(1, (pRight->val.nLen + 1) * TSDB_NCHAR_SIZE); - retVal = tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType, false); + // pRight->value.nLen + 1 is larger than the actual nchar string length + pColumnFilter->pz = (int64_t)calloc(1, bufLen * TSDB_NCHAR_SIZE); + retVal = tVariantDump(&pRight->value, (char*)pColumnFilter->pz, colType, false); size_t len = twcslen((wchar_t*)pColumnFilter->pz); pColumnFilter->len = len * TSDB_NCHAR_SIZE; } else { - retVal = tVariantDump(&pRight->val, (char*)&pColumnFilter->lowerBndd, colType, false); + retVal = tVariantDump(&pRight->value, (char*)&pColumnFilter->lowerBndd, colType, false); } if (retVal != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - switch (pExpr->nSQLOptr) { + switch (pExpr->tokenId) { case TK_LE: pColumnFilter->upperRelOptr = TSDB_RELATION_LESS_EQUAL; break; @@ -3077,162 +3177,45 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, } typedef struct SCondExpr { - tSQLExpr* pTagCond; - tSQLExpr* pTimewindow; + tSqlExpr* pTagCond; + tSqlExpr* pTimewindow; - tSQLExpr* pColumnCond; + tSqlExpr* pColumnCond; - tSQLExpr* pTableCond; + tSqlExpr* pTableCond; int16_t relType; // relation between table name in expression and other tag // filter condition expression, TK_AND or TK_OR int16_t tableCondIndex; - tSQLExpr* pJoinExpr; // join condition + tSqlExpr* pJoinExpr; // join condition bool tsJoin; } SCondExpr; -static int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision); - -static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) { - if (pExpr->nSQLOptr == TK_ID) { // column name - strncpy(*str, pExpr->colInfo.z, pExpr->colInfo.n); - *str += pExpr->colInfo.n; - - } else if (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_STRING) { // value - *str += tVariantToString(&pExpr->val, *str); - - } else if (pExpr->nSQLOptr >= TK_COUNT && pExpr->nSQLOptr <= TK_AVG_IRATE) { - /* - * arithmetic expression of aggregation, such as count(ts) + count(ts) *2 - */ - strncpy(*str, pExpr->operand.z, pExpr->operand.n); - *str += pExpr->operand.n; - } else { // not supported operation - assert(false); - } - - return TSDB_CODE_SUCCESS; -} - -// pExpr->nSQLOptr == 0 while handling "is null" query -static bool isExprLeafNode(tSQLExpr* pExpr) { - return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && - (pExpr->nSQLOptr == 0 || pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || pExpr->nSQLOptr == TK_SET); -} - -static bool isExprDirectParentOfLeafNode(tSQLExpr* pExpr) { - return (pExpr->pLeft != NULL && pExpr->pRight != NULL) && - (isExprLeafNode(pExpr->pLeft) && isExprLeafNode(pExpr->pRight)); -} - -static int32_t tSQLExprLeafToString(tSQLExpr* pExpr, bool addParentheses, char** output) { - if (!isExprDirectParentOfLeafNode(pExpr)) { - return TSDB_CODE_TSC_INVALID_SQL; - } +static int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t timePrecision); - tSQLExpr* pLeft = pExpr->pLeft; - tSQLExpr* pRight = pExpr->pRight; +static int32_t tablenameListToString(tSqlExpr* pExpr, SStringBuilder* sb) { + SArray* pList = pExpr->pParam; - if (addParentheses) { - *(*output) = '('; - *output += 1; - } - - tSQLExprNodeToString(pLeft, output); - if (optrToString(pExpr, output) != TSDB_CODE_SUCCESS) { + int32_t size = (int32_t) taosArrayGetSize(pList); + if (size <= 0) { return TSDB_CODE_TSC_INVALID_SQL; } - tSQLExprNodeToString(pRight, output); - - if (addParentheses) { - *(*output) = ')'; - *output += 1; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { - const char* le = "<="; - const char* ge = ">="; - const char* ne = "<>"; - const char* likeOptr = "LIKE"; - - switch (pExpr->nSQLOptr) { - case TK_LE: { - *(int16_t*)(*exprString) = *(int16_t*)le; - *exprString += 1; - break; - } - case TK_GE: { - *(int16_t*)(*exprString) = *(int16_t*)ge; - *exprString += 1; - break; - } - case TK_NE: { - *(int16_t*)(*exprString) = *(int16_t*)ne; - *exprString += 1; - break; - } - - case TK_LT: - *(*exprString) = '<'; - break; - case TK_GT: - *(*exprString) = '>'; - break; - case TK_EQ: - *(*exprString) = '='; - break; - case TK_PLUS: - *(*exprString) = '+'; - break; - case TK_MINUS: - *(*exprString) = '-'; - break; - case TK_STAR: - *(*exprString) = '*'; - break; - case TK_DIVIDE: - *(*exprString) = '/'; - break; - case TK_REM: - *(*exprString) = '%'; - break; - case TK_LIKE: { - int32_t len = sprintf(*exprString, " %s ", likeOptr); - *exprString += (len - 1); - break; - } - default: - return TSDB_CODE_TSC_INVALID_SQL; - } - - *exprString += 1; - - return TSDB_CODE_SUCCESS; -} - -static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) { - tSQLExprList* pList = pExpr->pParam; - if (pList->nExpr <= 0) { - return TSDB_CODE_TSC_INVALID_SQL; - } - - if (pList->nExpr > 0) { + if (size > 0) { taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); } - for (int32_t i = 0; i < pList->nExpr; ++i) { - tSQLExpr* pSub = pList->a[i].pNode; - taosStringBuilderAppendStringLen(sb, pSub->val.pz, pSub->val.nLen); + for (int32_t i = 0; i < size; ++i) { + tSqlExprItem* pSub = taosArrayGet(pList, i); + tVariant* pVar = &pSub->pNode->value; + + taosStringBuilderAppendStringLen(sb, pVar->pz, pVar->nLen); - if (i < pList->nExpr - 1) { + if (i < size - 1) { taosStringBuilderAppendString(sb, TBNAME_LIST_SEP); } - if (pSub->val.nLen <= 0 || !tscValidateTableNameLength(pSub->val.nLen)) { + if (pVar->nLen <= 0 || !tscValidateTableNameLength(pVar->nLen)) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -3240,9 +3223,9 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, SStringBuilder* sb) { return TSDB_CODE_SUCCESS; } -static int32_t tablenameCondToString(tSQLExpr* pExpr, SStringBuilder* sb) { +static int32_t tablenameCondToString(tSqlExpr* pExpr, SStringBuilder* sb) { taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); - taosStringBuilderAppendString(sb, pExpr->val.pz); + taosStringBuilderAppendString(sb, pExpr->value.pz); return TSDB_CODE_SUCCESS; } @@ -3254,7 +3237,7 @@ enum { TSQL_EXPR_TBNAME = 3, }; -static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSQLExpr* pExpr, int32_t sqlOptr) { +static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SColumnIndex* pIndex, tSqlExpr* pExpr, int32_t sqlOptr) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -3296,21 +3279,22 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC ((pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) ? 1 : 0); if (pColFilter->filterstr) { - if (pExpr->nSQLOptr != TK_EQ - && pExpr->nSQLOptr != TK_NE - && pExpr->nSQLOptr != TK_ISNULL - && pExpr->nSQLOptr != TK_NOTNULL - && pExpr->nSQLOptr != TK_LIKE + if (pExpr->tokenId != TK_EQ + && pExpr->tokenId != TK_NE + && pExpr->tokenId != TK_ISNULL + && pExpr->tokenId != TK_NOTNULL + && pExpr->tokenId != TK_LIKE ) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } } else { - if (pExpr->nSQLOptr == TK_LIKE) { + if (pExpr->tokenId == TK_LIKE) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } if (pSchema->type == TSDB_DATA_TYPE_BOOL) { - if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE) { + int32_t t = pExpr->tokenId; + if (t != TK_EQ && t != TK_NE && t != TK_NOTNULL && t != TK_ISNULL) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -3320,59 +3304,16 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC return doExtractColumnFilterInfo(pCmd, pQueryInfo, pColFilter, pIndex, pExpr); } -static void relToString(tSQLExpr* pExpr, char** str) { - assert(pExpr->nSQLOptr == TK_AND || pExpr->nSQLOptr == TK_OR); - - const char* or = "OR"; - const char*and = "AND"; - - // if (pQueryInfo->tagCond.relType == TSQL_STABLE_QTYPE_COND) { - if (pExpr->nSQLOptr == TK_AND) { - strcpy(*str, and); - *str += strlen(and); - } else { - strcpy(*str, or); - *str += strlen(or); - } -} - -UNUSED_FUNC -static int32_t getTagCondString(tSQLExpr* pExpr, char** str) { - if (pExpr == NULL) { - return TSDB_CODE_SUCCESS; - } - - if (!isExprDirectParentOfLeafNode(pExpr)) { - *(*str) = '('; - *str += 1; - - int32_t ret = getTagCondString(pExpr->pLeft, str); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - relToString(pExpr, str); - - ret = getTagCondString(pExpr->pRight, str); - - *(*str) = ')'; - *str += 1; - - return ret; - } - - return tSQLExprLeafToString(pExpr, true, str); -} - -static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pTableCond, SStringBuilder* sb) { +static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pTableCond, SStringBuilder* sb) { const char* msg0 = "invalid table name list"; + const char* msg1 = "not string following like"; if (pTableCond == NULL) { return TSDB_CODE_SUCCESS; } - tSQLExpr* pLeft = pTableCond->pLeft; - tSQLExpr* pRight = pTableCond->pRight; + tSqlExpr* pLeft = pTableCond->pLeft; + tSqlExpr* pRight = pTableCond->pRight; if (!isTablenameToken(&pLeft->colInfo)) { return TSDB_CODE_TSC_INVALID_SQL; @@ -3380,9 +3321,13 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* int32_t ret = TSDB_CODE_SUCCESS; - if (pTableCond->nSQLOptr == TK_IN) { + if (pTableCond->tokenId == TK_IN) { ret = tablenameListToString(pRight, sb); - } else if (pTableCond->nSQLOptr == TK_LIKE) { + } else if (pTableCond->tokenId == TK_LIKE) { + if (pRight->tokenId != TK_STRING) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + ret = tablenameCondToString(pRight, sb); } @@ -3393,18 +3338,18 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return ret; } -static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr, int32_t relOptr) { +static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, int32_t relOptr) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeafNode(pExpr)) { // internal node - int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->nSQLOptr); + if (!tSqlExprIsParentOfLeaf(pExpr)) { // internal node + int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->tokenId); if (ret != TSDB_CODE_SUCCESS) { return ret; } - return getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->nSQLOptr); + return getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pRight, pExpr->tokenId); } else { // handle leaf node SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -3415,23 +3360,26 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ } } -static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { - const char* msg1 = "invalid join query condition"; +static int32_t checkAndSetJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { + int32_t code = 0; + const char* msg1 = "timestamp required for join tables"; const char* msg3 = "type of join columns must be identical"; const char* msg4 = "invalid column name in join condition"; + const char* msg5 = "only support one join tag for each table"; if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeafNode(pExpr)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + if (!tSqlExprIsParentOfLeaf(pExpr)) { + code = checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr->pLeft); + if (code) { + return code; + } + + return checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr->pRight); } - STagCond* pTagCond = &pQueryInfo->tagCond; - SJoinNode* pLeft = &pTagCond->joinInfo.left; - SJoinNode* pRight = &pTagCond->joinInfo.right; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -3440,9 +3388,28 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - pLeft->uid = pTableMetaInfo->pTableMeta->id.uid; - pLeft->tagColId = pTagSchema1->colId; - strcpy(pLeft->tableId, pTableMetaInfo->name); + assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); + + SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; + if (*leftNode == NULL) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + (*leftNode)->uid = pTableMetaInfo->pTableMeta->id.uid; + (*leftNode)->tagColId = pTagSchema1->colId; + + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + if (!tscColumnExists(pTableMetaInfo->tagColList, &index)) { + tscColumnListInsert(pTableMetaInfo->tagColList, &index); + if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + } + } + } + + int16_t leftIdx = index.tableIndex; + index = (SColumnIndex)COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pExpr->pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -3452,53 +3419,60 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - pRight->uid = pTableMetaInfo->pTableMeta->id.uid; - pRight->tagColId = pTagSchema2->colId; - strcpy(pRight->tableId, pTableMetaInfo->name); + assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); - if (pTagSchema1->type != pTagSchema2->type) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; + if (*rightNode == NULL) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - pTagCond->joinInfo.hasJoin = true; - return TSDB_CODE_SUCCESS; -} - -// todo error handle / such as and /or mixed with +/-/*/ -int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { - tSQLExpr* pLeft = pExpr->pLeft; - tSQLExpr* pRight = pExpr->pRight; + (*rightNode)->uid = pTableMetaInfo->pTableMeta->id.uid; + (*rightNode)->tagColId = pTagSchema2->colId; - *(*exprString)++ = '('; - - if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { - doArithmeticExprToString(pLeft, exprString); - } else { - int32_t ret = tSQLExprNodeToString(pLeft, exprString); - if (ret != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + index.columnIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + if (!tscColumnExists(pTableMetaInfo->tagColList, &index)) { + tscColumnListInsert(pTableMetaInfo->tagColList, &index); + if (taosArrayGetSize(pTableMetaInfo->tagColList) > 1) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + } } } - optrToString(pExpr, exprString); + int16_t rightIdx = index.tableIndex; - if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) { - doArithmeticExprToString(pRight, exprString); - } else { - int32_t ret = tSQLExprNodeToString(pRight, exprString); - if (ret != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } + if (pTagSchema1->type != pTagSchema2->type) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - *(*exprString)++ = ')'; - + if ((*leftNode)->tagJoin == NULL) { + (*leftNode)->tagJoin = taosArrayInit(2, sizeof(int16_t)); + } + + if ((*rightNode)->tagJoin == NULL) { + (*rightNode)->tagJoin = taosArrayInit(2, sizeof(int16_t)); + } + + taosArrayPush((*leftNode)->tagJoin, &rightIdx); + taosArrayPush((*rightNode)->tagJoin, &leftIdx); + + pQueryInfo->tagCond.joinInfo.hasJoin = true; + return TSDB_CODE_SUCCESS; + } -static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, - int32_t* type, uint64_t* uid) { - if (pExpr->nSQLOptr == TK_ID) { +static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { + if (pExpr == NULL) { + return TSDB_CODE_SUCCESS; + } + + return checkAndSetJoinCondInfo(pCmd, pQueryInfo, pExpr); +} + +static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, + int32_t* type, uint64_t* uid) { + if (pExpr->type == SQL_NODE_TABLE_COLUMN) { if (*type == NON_ARITHMEIC_EXPR) { *type = NORMAL_ARITHMETIC; } else if (*type == AGG_ARIGHTMEIC) { @@ -3520,9 +3494,10 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer } pList->ids[pList->num++] = index; - } else if (pExpr->nSQLOptr == TK_FLOAT && (isnan(pExpr->val.dKey) || isinf(pExpr->val.dKey))) { + } else if ((pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.dKey) || isinf(pExpr->value.dKey))) || + pExpr->tokenId == TK_NULL) { return TSDB_CODE_TSC_INVALID_SQL; - } else if (pExpr->nSQLOptr >= TK_COUNT && pExpr->nSQLOptr <= TK_AVG_IRATE) { + } else if (pExpr->type == SQL_NODE_SQLFUNCTION) { if (*type == NON_ARITHMEIC_EXPR) { *type = AGG_ARIGHTMEIC; } else if (*type == NORMAL_ARITHMETIC) { @@ -3535,6 +3510,11 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer // sql function list in selection clause. // Append the sqlExpr into exprList of pQueryInfo structure sequentially + pExpr->functionId = isValidFunction(pExpr->operand.z, pExpr->operand.n); + if (pExpr->functionId < 0) { + return TSDB_CODE_TSC_INVALID_SQL; + } + if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -3567,16 +3547,16 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer return TSDB_CODE_SUCCESS; } -static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { +static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - tSQLExpr* pLeft = pExpr->pLeft; + tSqlExpr* pLeft = pExpr->pLeft; uint64_t uidLeft = 0; uint64_t uidRight = 0; - if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { + if (pLeft->type == SQL_NODE_EXPR) { int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -3588,8 +3568,8 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI } } - tSQLExpr* pRight = pExpr->pRight; - if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) { + tSqlExpr* pRight = pExpr->pRight; + if (pRight->type == SQL_NODE_EXPR) { int32_t ret = validateArithmeticSQLExpr(pCmd, pRight, pQueryInfo, pList, type); if (ret != TSDB_CODE_SUCCESS) { return ret; @@ -3609,7 +3589,7 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI return TSDB_CODE_SUCCESS; } -static bool isValidExpr(tSQLExpr* pLeft, tSQLExpr* pRight, int32_t optr) { +static bool isValidExpr(tSqlExpr* pLeft, tSqlExpr* pRight, int32_t optr) { if (pLeft == NULL || (pRight == NULL && optr != TK_IN)) { return false; } @@ -3622,34 +3602,27 @@ static bool isValidExpr(tSQLExpr* pLeft, tSQLExpr* pRight, int32_t optr) { * * However, columnA < 4+12 is valid */ - if (pLeft->nSQLOptr >= TK_COUNT && pLeft->nSQLOptr <= TK_AVG_IRATE) { + if (pLeft->type == SQL_NODE_SQLFUNCTION) { return false; } if (pRight == NULL) { return true; } - - if (pRight->nSQLOptr >= TK_COUNT && pRight->nSQLOptr <= TK_AVG_IRATE) { - return false; - } - if (pLeft->nSQLOptr >= TK_BOOL - && pLeft->nSQLOptr <= TK_BINARY - && pRight->nSQLOptr >= TK_BOOL - && pRight->nSQLOptr <= TK_BINARY) { + if (pLeft->tokenId >= TK_BOOL && pLeft->tokenId <= TK_BINARY && pRight->tokenId >= TK_BOOL && pRight->tokenId <= TK_BINARY) { return false; } return true; } -static void exchangeExpr(tSQLExpr* pExpr) { - tSQLExpr* pLeft = pExpr->pLeft; - tSQLExpr* pRight = pExpr->pRight; +static void exchangeExpr(tSqlExpr* pExpr) { + tSqlExpr* pLeft = pExpr->pLeft; + tSqlExpr* pRight = pExpr->pRight; - if (pRight->nSQLOptr == TK_ID && (pLeft->nSQLOptr == TK_INTEGER || pLeft->nSQLOptr == TK_FLOAT || - pLeft->nSQLOptr == TK_STRING || pLeft->nSQLOptr == TK_BOOL)) { + if (pRight->tokenId == TK_ID && (pLeft->tokenId == TK_INTEGER || pLeft->tokenId == TK_FLOAT || + pLeft->tokenId == TK_STRING || pLeft->tokenId == TK_BOOL)) { /* * exchange value of the left handside and the value of the right-handside * to make sure that the value of filter expression always locates in @@ -3657,7 +3630,7 @@ static void exchangeExpr(tSQLExpr* pExpr) { * the column-id is at the left handside. */ uint32_t optr = 0; - switch (pExpr->nSQLOptr) { + switch (pExpr->tokenId) { case TK_LE: optr = TK_GE; break; @@ -3671,28 +3644,28 @@ static void exchangeExpr(tSQLExpr* pExpr) { optr = TK_LE; break; default: - optr = pExpr->nSQLOptr; + optr = pExpr->tokenId; } - pExpr->nSQLOptr = optr; + pExpr->tokenId = optr; SWAP(pExpr->pLeft, pExpr->pRight, void*); } } -static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr, SColumnIndex* pLeftIndex) { +static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr, SColumnIndex* pLeftIndex) { const char* msg1 = "illegal column name"; const char* msg2 = "= is expected in join expression"; const char* msg3 = "join column must have same type"; const char* msg4 = "self join is not allowed"; const char* msg5 = "join table must be the same type(table to table, super table to super table)"; - tSQLExpr* pRight = pExpr->pRight; + tSqlExpr* pRight = pExpr->pRight; - if (pRight->nSQLOptr != TK_ID) { + if (pRight->tokenId != TK_ID) { return true; } - if (pExpr->nSQLOptr != TK_EQ) { + if (pExpr->tokenId != TK_EQ) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); return false; } @@ -3730,11 +3703,11 @@ static bool validateJoinExprNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr return true; } -static bool validTableNameOptr(tSQLExpr* pExpr) { +static bool validTableNameOptr(tSqlExpr* pExpr) { const char nameFilterOptr[] = {TK_IN, TK_LIKE}; for (int32_t i = 0; i < tListLen(nameFilterOptr); ++i) { - if (pExpr->nSQLOptr == nameFilterOptr[i]) { + if (pExpr->tokenId == nameFilterOptr[i]) { return true; } } @@ -3742,7 +3715,7 @@ static bool validTableNameOptr(tSQLExpr* pExpr) { return false; } -static int32_t setExprToCond(tSQLExpr** parent, tSQLExpr* pExpr, const char* msg, int32_t parentOptr, char* msgBuf) { +static int32_t setExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, const char* msg, int32_t parentOptr, char* msgBuf) { if (*parent != NULL) { if (parentOptr == TK_OR && msg != NULL) { return invalidSqlErrMsg(msgBuf, msg); @@ -3756,19 +3729,51 @@ static int32_t setExprToCond(tSQLExpr** parent, tSQLExpr* pExpr, const char* msg return TSDB_CODE_SUCCESS; } -static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SCondExpr* pCondExpr, +static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) { + const char* msg = "only support is [not] null"; + + tSqlExpr* pRight = pExpr->pRight; + if (pRight->tokenId == TK_NULL && (!(pExpr->tokenId == TK_ISNULL || pExpr->tokenId == TK_NOTNULL))) { + return invalidSqlErrMsg(msgBuf, msg); + } + + return TSDB_CODE_SUCCESS; +} + +// check for like expression +static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { + const char* msg1 = "wildcard string should be less than 20 characters"; + const char* msg2 = "illegal column name"; + + tSqlExpr* pLeft = pExpr->pLeft; + tSqlExpr* pRight = pExpr->pRight; + + if (pExpr->tokenId == TK_LIKE) { + if (pRight->value.nLen > TSDB_PATTERN_STRING_MAX_LEN) { + return invalidSqlErrMsg(msgBuf, msg1); + } + + SSchema* pSchema = tscGetTableSchema(pTableMeta); + if ((!isTablenameToken(&pLeft->colInfo)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) { + return invalidSqlErrMsg(msgBuf, msg2); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, int32_t* type, int32_t parentOptr) { const char* msg1 = "table query cannot use tags filter"; const char* msg2 = "illegal column name"; const char* msg3 = "only one query time range allowed"; - const char* msg4 = "only one join condition allowed"; + const char* msg4 = "too many join tables"; const char* msg5 = "not support ordinary column join"; const char* msg6 = "only one query condition on tbname allowed"; const char* msg7 = "only in/like allowed in filter table name"; - const char* msg8 = "wildcard string should be less than 20 characters"; - - tSQLExpr* pLeft = (*pExpr)->pLeft; - tSQLExpr* pRight = (*pExpr)->pRight; + + tSqlExpr* pLeft = (*pExpr)->pLeft; + tSqlExpr* pRight = (*pExpr)->pRight; int32_t ret = TSDB_CODE_SUCCESS; @@ -3777,21 +3782,73 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - assert(isExprDirectParentOfLeafNode(*pExpr)); + assert(tSqlExprIsParentOfLeaf(*pExpr)); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + // validate the null expression + int32_t code = validateNullExpr(*pExpr, tscGetErrorMsgPayload(pCmd)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // validate the like expression + code = validateLikeExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_SQL; } // set join query condition - if (pRight->nSQLOptr == TK_ID) { // no need to keep the timestamp join condition + if (pRight->tokenId == TK_ID) { // no need to keep the timestamp join condition TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_QUERY); pCondExpr->tsJoin = true; + assert(index.tableIndex >= 0 && index.tableIndex < TSDB_MAX_JOIN_TABLE_NUM); + SJoinNode **leftNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; + if (*leftNode == NULL) { + *leftNode = calloc(1, sizeof(SJoinNode)); + if (*leftNode == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + int16_t leftIdx = index.tableIndex; + + if (getColumnIndexByName(pCmd, &pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + if (index.tableIndex < 0 || index.tableIndex >= TSDB_MAX_JOIN_TABLE_NUM) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + + SJoinNode **rightNode = &pQueryInfo->tagCond.joinInfo.joinTables[index.tableIndex]; + if (*rightNode == NULL) { + *rightNode = calloc(1, sizeof(SJoinNode)); + if (*rightNode == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + int16_t rightIdx = index.tableIndex; + + if ((*leftNode)->tsJoin == NULL) { + (*leftNode)->tsJoin = taosArrayInit(2, sizeof(int16_t)); + } + + if ((*rightNode)->tsJoin == NULL) { + (*rightNode)->tsJoin = taosArrayInit(2, sizeof(int16_t)); + } + + taosArrayPush((*leftNode)->tsJoin, &rightIdx); + taosArrayPush((*rightNode)->tsJoin, &leftIdx); + /* * to release expression, e.g., m1.ts = m2.ts, * since this expression is used to set the join query type @@ -3809,20 +3866,6 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - // check for like expression - if ((*pExpr)->nSQLOptr == TK_LIKE) { - if (pRight->val.nLen > TSDB_PATTERN_STRING_MAX_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - if ((!isTablenameToken(&pLeft->colInfo)) && pSchema[index.columnIndex].type != TSDB_DATA_TYPE_BINARY && - pSchema[index.columnIndex].type != TSDB_DATA_TYPE_NCHAR) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - // in case of in operator, keep it in a seprate attribute if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (!validTableNameOptr(*pExpr)) { @@ -3844,15 +3887,11 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL *type = TSQL_EXPR_TBNAME; *pExpr = NULL; } else { - if (pRight != NULL && pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query + if (pRight != NULL && pRight->tokenId == TK_ID) { // join on tag columns for stable query if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_SQL; } - if (pCondExpr->pJoinExpr != NULL) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - pQueryInfo->type |= TSDB_QUERY_TYPE_JOIN_QUERY; ret = setExprToCond(&pCondExpr->pJoinExpr, *pExpr, NULL, parentOptr, pQueryInfo->msg); *pExpr = NULL; @@ -3868,7 +3907,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL } else { // query on other columns *type = TSQL_EXPR_COLUMN; - if (pRight->nSQLOptr == TK_ID) { // other column cannot be served as the join column + if (pRight->tokenId == TK_ID) { // other column cannot be served as the join column return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } @@ -3879,7 +3918,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL return ret; } -int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SCondExpr* pCondExpr, +int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, int32_t* type, int32_t parentOptr) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; @@ -3887,23 +3926,23 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr const char* msg1 = "query condition between different columns must use 'AND'"; - tSQLExpr* pLeft = (*pExpr)->pLeft; - tSQLExpr* pRight = (*pExpr)->pRight; + tSqlExpr* pLeft = (*pExpr)->pLeft; + tSqlExpr* pRight = (*pExpr)->pRight; - if (!isValidExpr(pLeft, pRight, (*pExpr)->nSQLOptr)) { + if (!isValidExpr(pLeft, pRight, (*pExpr)->tokenId)) { return TSDB_CODE_TSC_INVALID_SQL; } int32_t leftType = -1; int32_t rightType = -1; - if (!isExprDirectParentOfLeafNode(*pExpr)) { - int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->nSQLOptr); + if (!tSqlExprIsParentOfLeaf(*pExpr)) { + int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->tokenId); if (ret != TSDB_CODE_SUCCESS) { return ret; } - ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->nSQLOptr); + ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pRight, pCondExpr, &rightType, (*pExpr)->tokenId); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -3913,7 +3952,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr * expression is not valid for parent node, it must be TK_AND operator. */ if (leftType != rightType) { - if ((*pExpr)->nSQLOptr == TK_OR && (leftType + rightType != TSQL_EXPR_TBNAME + TSQL_EXPR_TAG)) { + if ((*pExpr)->tokenId == TK_OR && (leftType + rightType != TSQL_EXPR_TBNAME + TSQL_EXPR_TAG)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -3927,40 +3966,9 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr return handleExprInQueryCond(pCmd, pQueryInfo, pExpr, pCondExpr, type, parentOptr); } -static void doCompactQueryExpr(tSQLExpr** pExpr) { - if (*pExpr == NULL || isExprDirectParentOfLeafNode(*pExpr)) { - return; - } - - if ((*pExpr)->pLeft) { - doCompactQueryExpr(&(*pExpr)->pLeft); - } - - if ((*pExpr)->pRight) { - doCompactQueryExpr(&(*pExpr)->pRight); - } - - if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL && - ((*pExpr)->nSQLOptr == TK_OR || (*pExpr)->nSQLOptr == TK_AND)) { - tSqlExprNodeDestroy(*pExpr); - *pExpr = NULL; - - } else if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight != NULL) { - tSQLExpr* tmpPtr = (*pExpr)->pRight; - tSqlExprNodeDestroy(*pExpr); - - (*pExpr) = tmpPtr; - } else if ((*pExpr)->pRight == NULL && (*pExpr)->pLeft != NULL) { - tSQLExpr* tmpPtr = (*pExpr)->pLeft; - tSqlExprNodeDestroy(*pExpr); - - (*pExpr) = tmpPtr; - } -} - -static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* pQueryInfo, tSQLExpr** pOut, int32_t tableIndex) { - if (isExprDirectParentOfLeafNode(*pExpr)) { - tSQLExpr* pLeft = (*pExpr)->pLeft; +static void doExtractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, tSqlExpr** pOut, int32_t tableIndex) { + if (tSqlExprIsParentOfLeaf(*pExpr)) { + tSqlExpr* pLeft = (*pExpr)->pLeft; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pLeft->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -3975,19 +3983,19 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* (*pExpr) = NULL; } else { - *pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->nSQLOptr); + *pOut = tSqlExprCreate(NULL, NULL, (*pExpr)->tokenId); doExtractExprForSTable(pCmd, &(*pExpr)->pLeft, pQueryInfo, &((*pOut)->pLeft), tableIndex); doExtractExprForSTable(pCmd, &(*pExpr)->pRight, pQueryInfo, &((*pOut)->pRight), tableIndex); } } -static tSQLExpr* extractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex) { - tSQLExpr* pResExpr = NULL; +static tSqlExpr* extractExprForSTable(SSqlCmd* pCmd, tSqlExpr** pExpr, SQueryInfo* pQueryInfo, int32_t tableIndex) { + tSqlExpr* pResExpr = NULL; if (*pExpr != NULL) { doExtractExprForSTable(pCmd, pExpr, pQueryInfo, &pResExpr, tableIndex); - doCompactQueryExpr(&pResExpr); + tSqlExprCompact(&pResExpr); } return pResExpr; @@ -4007,7 +4015,7 @@ int tableNameCompar(const void* lhs, const void* rhs) { } static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, const char* account, - tSQLExpr* pExpr, int16_t tableCondIndex, SStringBuilder* sb) { + tSqlExpr* pExpr, int16_t tableCondIndex, SStringBuilder* sb) { const char* msg = "table name too long"; if (pExpr == NULL) { @@ -4019,9 +4027,9 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, STagCond* pTagCond = &pQueryInfo->tagCond; pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid; - assert(pExpr->nSQLOptr == TK_LIKE || pExpr->nSQLOptr == TK_IN); + assert(pExpr->tokenId == TK_LIKE || pExpr->tokenId == TK_IN); - if (pExpr->nSQLOptr == TK_LIKE) { + if (pExpr->tokenId == TK_LIKE) { char* str = taosStringBuilderGetResult(sb, NULL); pQueryInfo->tagCond.tbnameCond.cond = strdup(str); pQueryInfo->tagCond.tbnameCond.len = (int32_t) strlen(str); @@ -4031,8 +4039,6 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1)); taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); - char db[TSDB_TABLE_FNAME_LEN] = {0}; - // remove the duplicated input table names int32_t num = 0; char* tableNameString = taosStringBuilderGetResult(sb, NULL); @@ -4048,7 +4054,8 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, } num = j; - char* name = extractDBName(pTableMetaInfo->name, db); + char name[TSDB_DB_NAME_LEN] = {0}; + tNameGetDbName(&pTableMetaInfo->name, name); SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; for (int32_t i = 0; i < num; ++i) { @@ -4109,21 +4116,25 @@ static bool validateFilterExpr(SQueryInfo* pQueryInfo) { return true; } -static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { +static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr* pExpr) { const char* msg0 = "invalid timestamp"; const char* msg1 = "only one time stamp window allowed"; - + int32_t code = 0; + if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - if (!isExprDirectParentOfLeafNode(pExpr)) { - if (pExpr->nSQLOptr == TK_OR) { + if (!tSqlExprIsParentOfLeaf(pExpr)) { + if (pExpr->tokenId == TK_OR) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft); - + code = getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pLeft); + if (code) { + return code; + } + return getTimeRangeFromExpr(pCmd, pQueryInfo, pExpr->pRight); } else { SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -4134,10 +4145,10 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLE STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - tSQLExpr* pRight = pExpr->pRight; + tSqlExpr* pRight = pExpr->pRight; STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; - if (getTimeRange(&win, pRight, pExpr->nSQLOptr, tinfo.precision) != TSDB_CODE_SUCCESS) { + if (getTimeRange(&win, pRight, pExpr->tokenId, tinfo.precision) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } @@ -4204,6 +4215,7 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { } } +/* static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); if (QUERY_IS_JOIN_QUERY(pQueryInfo->type) && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { @@ -4226,8 +4238,88 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SQueryInfo* pQueryInf tscColumnListInsert(pTableMetaInfo->tagColList, &index); } } +*/ + +static int32_t validateTagCondExpr(SSqlCmd* pCmd, tExprNode *p) { + const char *msg1 = "invalid tag operator"; + const char* msg2 = "not supported filter condition"; + + do { + if (p->nodeType != TSQL_NODE_EXPR) { + break; + } + + if (!p->_node.pLeft || !p->_node.pRight) { + break; + } + + if (IS_ARITHMETIC_OPTR(p->_node.optr)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + if (!IS_RELATION_OPTR(p->_node.optr)) { + break; + } + + tVariant * vVariant = NULL; + int32_t schemaType = -1; + + if (p->_node.pLeft->nodeType == TSQL_NODE_VALUE && p->_node.pRight->nodeType == TSQL_NODE_COL) { + if (!p->_node.pRight->pSchema) { + break; + } + + vVariant = p->_node.pLeft->pVal; + schemaType = p->_node.pRight->pSchema->type; + } else if (p->_node.pLeft->nodeType == TSQL_NODE_COL && p->_node.pRight->nodeType == TSQL_NODE_VALUE) { + if (!p->_node.pLeft->pSchema) { + break; + } + + vVariant = p->_node.pRight->pVal; + schemaType = p->_node.pLeft->pSchema->type; + } else { + break; + } + + if (schemaType >= TSDB_DATA_TYPE_TINYINT && schemaType <= TSDB_DATA_TYPE_BIGINT) { + schemaType = TSDB_DATA_TYPE_BIGINT; + } else if (schemaType == TSDB_DATA_TYPE_FLOAT || schemaType == TSDB_DATA_TYPE_DOUBLE) { + schemaType = TSDB_DATA_TYPE_DOUBLE; + } + + int32_t retVal = TSDB_CODE_SUCCESS; + + int32_t bufLen = 0; + if (IS_NUMERIC_TYPE(vVariant->nType)) { + bufLen = 60; // The maximum length of string that a number is converted to. + } else { + bufLen = vVariant->nLen + 1; + } + + if (schemaType == TSDB_DATA_TYPE_BINARY) { + char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + retVal = tVariantDump(vVariant, tmp, schemaType, false); + free(tmp); + } else if (schemaType == TSDB_DATA_TYPE_NCHAR) { + // pRight->value.nLen + 1 is larger than the actual nchar string length + char *tmp = calloc(1, bufLen * TSDB_NCHAR_SIZE); + retVal = tVariantDump(vVariant, tmp, schemaType, false); + free(tmp); + } else { + double tmp; + retVal = tVariantDump(vVariant, (char*)&tmp, schemaType, false); + } + + if (retVal != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + } while (0); + + return TSDB_CODE_SUCCESS; +} -static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSQLExpr** pExpr) { +static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondExpr* pCondExpr, tSqlExpr** pExpr) { int32_t ret = TSDB_CODE_SUCCESS; if (pCondExpr->pTagCond == NULL) { @@ -4235,7 +4327,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE } for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - tSQLExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i); + tSqlExpr* p1 = extractExprForSTable(pCmd, pExpr, pQueryInfo, i); if (p1 == NULL) { // no query condition on this table continue; } @@ -4267,22 +4359,126 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE } tsSetSTableQueryCond(&pQueryInfo->tagCond, uid, &bw); - doCompactQueryExpr(pExpr); + tSqlExprCompact(pExpr); + + if (ret == TSDB_CODE_SUCCESS) { + ret = validateTagCondExpr(pCmd, p); + } tSqlExprDestroy(p1); - tExprTreeDestroy(&p, NULL); + tExprTreeDestroy(p, NULL); taosArrayDestroy(colList); if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); } + + if (ret) { + break; + } } pCondExpr->pTagCond = NULL; return ret; } -int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql) { +int32_t validateJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { + const char* msg1 = "timestamp required for join tables"; + const char* msg2 = "tag required for join stables"; + + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + SJoinNode *node = pQueryInfo->tagCond.joinInfo.joinTables[i]; + + if (node == NULL || node->tsJoin == NULL || taosArrayGetSize(node->tsJoin) <= 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); + } + } + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + SJoinNode *node = pQueryInfo->tagCond.joinInfo.joinTables[i]; + + if (node == NULL || node->tagJoin == NULL || taosArrayGetSize(node->tagJoin) <= 0) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); + } + } + } + + return TSDB_CODE_SUCCESS; +} + + +void mergeJoinNodesImpl(int8_t* r, int8_t* p, int16_t* tidx, SJoinNode** nodes, int32_t type) { + SJoinNode *node = nodes[*tidx]; + SArray* arr = (type == 0) ? node->tsJoin : node->tagJoin; + size_t size = taosArrayGetSize(arr); + + p[*tidx] = 1; + + for (int32_t j = 0; j < size; j++) { + int16_t* idx = taosArrayGet(arr, j); + r[*idx] = 1; + if (p[*idx] == 0) { + mergeJoinNodesImpl(r, p, idx, nodes, type); + } + } +} + +int32_t mergeJoinNodes(SQueryInfo* pQueryInfo, SSqlObj* pSql) { + const char* msg1 = "not all join tables have same timestamp"; + const char* msg2 = "not all join tables have same tag"; + + int8_t r[TSDB_MAX_JOIN_TABLE_NUM] = {0}; + int8_t p[TSDB_MAX_JOIN_TABLE_NUM] = {0}; + + for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { + mergeJoinNodesImpl(r, p, &i, pQueryInfo->tagCond.joinInfo.joinTables, 0); + + taosArrayClear(pQueryInfo->tagCond.joinInfo.joinTables[i]->tsJoin); + + for (int32_t j = 0; j < TSDB_MAX_JOIN_TABLE_NUM; ++j) { + if (r[j]) { + taosArrayPush(pQueryInfo->tagCond.joinInfo.joinTables[i]->tsJoin, &j); + } + } + + memset(r, 0, sizeof(r)); + memset(p, 0, sizeof(p)); + } + + if (taosArrayGetSize(pQueryInfo->tagCond.joinInfo.joinTables[0]->tsJoin) != pQueryInfo->numOfTables) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg1); + } + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + for (int16_t i = 0; i < pQueryInfo->numOfTables; ++i) { + mergeJoinNodesImpl(r, p, &i, pQueryInfo->tagCond.joinInfo.joinTables, 1); + + taosArrayClear(pQueryInfo->tagCond.joinInfo.joinTables[i]->tagJoin); + + for (int32_t j = 0; j < TSDB_MAX_JOIN_TABLE_NUM; ++j) { + if (r[j]) { + taosArrayPush(pQueryInfo->tagCond.joinInfo.joinTables[i]->tagJoin, &j); + } + } + + memset(r, 0, sizeof(r)); + memset(p, 0, sizeof(p)); + } + + if (taosArrayGetSize(pQueryInfo->tagCond.joinInfo.joinTables[0]->tagJoin) != pQueryInfo->numOfTables) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); + } + + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } @@ -4301,11 +4497,11 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql } int32_t type = 0; - if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->nSQLOptr)) != TSDB_CODE_SUCCESS) { + if ((ret = getQueryCondExpr(&pSql->cmd, pQueryInfo, pExpr, &condExpr, &type, (*pExpr)->tokenId)) != TSDB_CODE_SUCCESS) { return ret; } - doCompactQueryExpr(pExpr); + tSqlExprCompact(pExpr); // after expression compact, the expression tree is only include tag query condition condExpr.pTagCond = (*pExpr); @@ -4327,17 +4523,17 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql // 4. get the table name query condition if ((ret = getTablenameCond(&pSql->cmd, pQueryInfo, condExpr.pTableCond, &sb)) != TSDB_CODE_SUCCESS) { - return ret; + goto PARSE_WHERE_EXIT; } // 5. other column query condition if ((ret = getColumnQueryCondInfo(&pSql->cmd, pQueryInfo, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { - return ret; + goto PARSE_WHERE_EXIT; } // 6. join condition if ((ret = getJoinCondInfo(&pSql->cmd, pQueryInfo, condExpr.pJoinExpr)) != TSDB_CODE_SUCCESS) { - return ret; + goto PARSE_WHERE_EXIT; } // 7. query condition for table name @@ -4345,20 +4541,37 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql ret = setTableCondForSTableQuery(&pSql->cmd, pQueryInfo, getAccountId(pSql), condExpr.pTableCond, condExpr.tableCondIndex, &sb); taosStringBuilderDestroy(&sb); + if (ret) { + goto PARSE_WHERE_EXIT; + } if (!validateFilterExpr(pQueryInfo)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); + ret = invalidSqlErrMsg(tscGetErrorMsgPayload(&pSql->cmd), msg2); + goto PARSE_WHERE_EXIT; } - doAddJoinTagsColumnsIntoTagList(&pSql->cmd, pQueryInfo, &condExpr); + //doAddJoinTagsColumnsIntoTagList(&pSql->cmd, pQueryInfo, &condExpr); + if (condExpr.tsJoin) { + ret = validateJoinNodes(pQueryInfo, pSql); + if (ret) { + goto PARSE_WHERE_EXIT; + } + + ret = mergeJoinNodes(pQueryInfo, pSql); + if (ret) { + goto PARSE_WHERE_EXIT; + } + } + +PARSE_WHERE_EXIT: cleanQueryExpr(&condExpr); return ret; } -int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t timePrecision) { +int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t timePrecision) { // this is join condition, do nothing - if (pRight->nSQLOptr == TK_ID) { + if (pRight->tokenId == TK_ID) { return TSDB_CODE_SUCCESS; } @@ -4366,42 +4579,42 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t * filter primary ts filter expression like: * where ts in ('2015-12-12 4:8:12') */ - if (pRight->nSQLOptr == TK_SET || optr == TK_IN) { + if (pRight->tokenId == TK_SET || optr == TK_IN) { return TSDB_CODE_TSC_INVALID_SQL; } int64_t val = 0; bool parsed = false; - if (pRight->val.nType == TSDB_DATA_TYPE_BINARY) { - pRight->val.nLen = strdequote(pRight->val.pz); + if (pRight->value.nType == TSDB_DATA_TYPE_BINARY) { + pRight->value.nLen = strdequote(pRight->value.pz); - char* seg = strnchr(pRight->val.pz, '-', pRight->val.nLen, false); + char* seg = strnchr(pRight->value.pz, '-', pRight->value.nLen, false); if (seg != NULL) { - if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO, tsDaylight) == TSDB_CODE_SUCCESS) { + if (taosParseTime(pRight->value.pz, &val, pRight->value.nLen, TSDB_TIME_PRECISION_MICRO, tsDaylight) == TSDB_CODE_SUCCESS) { parsed = true; } else { return TSDB_CODE_TSC_INVALID_SQL; } } else { - SStrToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; - int32_t len = tSQLGetToken(pRight->val.pz, &token.type); + SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID}; + int32_t len = tSQLGetToken(pRight->value.pz, &token.type); - if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->val.nLen) { + if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) { return TSDB_CODE_TSC_INVALID_SQL; } } - } else if (pRight->nSQLOptr == TK_INTEGER && timePrecision == TSDB_TIME_PRECISION_MILLI) { + } else if (pRight->tokenId == TK_INTEGER && timePrecision == TSDB_TIME_PRECISION_MILLI) { /* - * if the pRight->nSQLOptr == TK_INTEGER/TK_FLOAT, the value is adaptive, we + * if the pRight->tokenId == TK_INTEGER/TK_FLOAT, the value is adaptive, we * need the time precision in metermeta to transfer the value in MICROSECOND * * Additional check to avoid data overflow */ - if (pRight->val.i64 <= INT64_MAX / 1000) { - pRight->val.i64 *= 1000; + if (pRight->value.i64 <= INT64_MAX / 1000) { + pRight->value.i64 *= 1000; } - } else if (pRight->nSQLOptr == TK_FLOAT && timePrecision == TSDB_TIME_PRECISION_MILLI) { - pRight->val.dKey *= 1000; + } else if (pRight->tokenId == TK_FLOAT && timePrecision == TSDB_TIME_PRECISION_MILLI) { + pRight->value.dKey *= 1000; } if (!parsed) { @@ -4409,7 +4622,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t * failed to parse timestamp in regular formation, try next * it may be a epoch time in string format */ - tVariantDump(&pRight->val, (char*)&val, TSDB_DATA_TYPE_BIGINT, true); + tVariantDump(&pRight->value, (char*)&val, TSDB_DATA_TYPE_BIGINT, true); /* * transfer it into MICROSECOND format if it is a string, since for @@ -4417,7 +4630,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t * * additional check to avoid data overflow */ - if (pRight->nSQLOptr == TK_STRING && timePrecision == TSDB_TIME_PRECISION_MILLI) { + if (pRight->tokenId == TK_STRING && timePrecision == TSDB_TIME_PRECISION_MILLI) { if (val <= INT64_MAX / 1000) { val *= 1000; } @@ -4476,7 +4689,7 @@ int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return TSDB_CODE_SUCCESS; } -int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) { +int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySqlNode* pQuerySQL) { SArray* pFillToken = pQuerySQL->fillType; tVariantListItem* pItem = taosArrayGet(pFillToken, 0); @@ -4491,10 +4704,10 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - size_t size = tscNumOfFields(pQueryInfo); + size_t numOfFields = tscNumOfFields(pQueryInfo); if (pQueryInfo->fillVal == NULL) { - pQueryInfo->fillVal = calloc(size, sizeof(int64_t)); + pQueryInfo->fillVal = calloc(numOfFields, sizeof(int64_t)); if (pQueryInfo->fillVal == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -4504,19 +4717,21 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery pQueryInfo->fillType = TSDB_FILL_NONE; } else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) { pQueryInfo->fillType = TSDB_FILL_NULL; - for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) { + for (int32_t i = START_INTERPO_COL_IDX; i < numOfFields; ++i) { TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); setNull((char*)&pQueryInfo->fillVal[i], pField->type, pField->bytes); } } else if (strncasecmp(pItem->pVar.pz, "prev", 4) == 0 && pItem->pVar.nLen == 4) { pQueryInfo->fillType = TSDB_FILL_PREV; + } else if (strncasecmp(pItem->pVar.pz, "next", 4) == 0 && pItem->pVar.nLen == 4) { + pQueryInfo->fillType = TSDB_FILL_NEXT; } else if (strncasecmp(pItem->pVar.pz, "linear", 6) == 0 && pItem->pVar.nLen == 6) { pQueryInfo->fillType = TSDB_FILL_LINEAR; } else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) { pQueryInfo->fillType = TSDB_FILL_SET_VALUE; size_t num = taosArrayGetSize(pFillToken); - if (num == 1) { + if (num == 1) { // no actual value, return with error code return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4527,11 +4742,11 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery if (tscIsPointInterpQuery(pQueryInfo)) { startPos = 0; - if (numOfFillVal > size) { - numOfFillVal = (int32_t)size; + if (numOfFillVal > numOfFields) { + numOfFillVal = (int32_t)numOfFields; } } else { - numOfFillVal = (int16_t)((num > (int32_t)size) ? (int32_t)size : num); + numOfFillVal = (int16_t)((num > (int32_t)numOfFields) ? (int32_t)numOfFields : num); } int32_t j = 1; @@ -4551,10 +4766,10 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery } } - if ((num < size) || ((num - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) { + if ((num < numOfFields) || ((num - 1 < numOfFields) && (tscIsPointInterpQuery(pQueryInfo)))) { tVariantListItem* lastItem = taosArrayGetLast(pFillToken); - for (int32_t i = numOfFillVal; i < size; ++i) { + for (int32_t i = numOfFillVal; i < numOfFields; ++i) { TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) { @@ -4596,20 +4811,26 @@ static void setDefaultOrderInfo(SQueryInfo* pQueryInfo) { } } -int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql, SSchema* pSchema) { +int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySqlNode* pQuerySqlNode, SSchema* pSchema) { const char* msg0 = "only support order by primary timestamp"; const char* msg1 = "invalid column name"; - const char* msg2 = "only support order by primary timestamp or first tag in groupby clause allowed"; + const char* msg2 = "order by primary timestamp or first tag in groupby clause allowed"; const char* msg3 = "invalid column in order by clause, only primary timestamp or first tag in groupby clause allowed"; setDefaultOrderInfo(pQueryInfo); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pQuerySql->pSortOrder == NULL) { + + if (pQueryInfo->distinctTag == true) { + pQueryInfo->order.order = TSDB_ORDER_ASC; + pQueryInfo->order.orderColId = 0; + return TSDB_CODE_SUCCESS; + } + if (pQuerySqlNode->pSortOrder == NULL) { return TSDB_CODE_SUCCESS; } - SArray* pSortorder = pQuerySql->pSortOrder; + SArray* pSortorder = pQuerySqlNode->pSortOrder; /* * for table query, there is only one or none order option is allowed, which is the @@ -4677,7 +4898,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu if (orderByTags) { pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0); + tVariantListItem* p1 = taosArrayGet(pQuerySqlNode->pSortOrder, 0); pQueryInfo->groupbyExpr.orderType = p1->sortOrder; } else if (isTopBottomQuery(pQueryInfo)) { /* order of top/bottom query in interval is not valid */ @@ -4689,12 +4910,12 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0); + tVariantListItem* p1 = taosArrayGet(pQuerySqlNode->pSortOrder, 0); pQueryInfo->order.order = p1->sortOrder; pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; return TSDB_CODE_SUCCESS; } else { - tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0); + tVariantListItem* p1 = taosArrayGet(pQuerySqlNode->pSortOrder, 0); pQueryInfo->order.order = p1->sortOrder; pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; @@ -4707,7 +4928,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu } if (s == 2) { - tVariantListItem *pItem = taosArrayGet(pQuerySql->pSortOrder, 0); + tVariantListItem *pItem = taosArrayGet(pQuerySqlNode->pSortOrder, 0); if (orderByTags) { pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta); pQueryInfo->groupbyExpr.orderType = pItem->sortOrder; @@ -4716,7 +4937,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX; } - pItem = taosArrayGet(pQuerySql->pSortOrder, 1); + pItem = taosArrayGet(pQuerySqlNode->pSortOrder, 1); tVariant* pVar2 = &pItem->pVar; SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -4751,13 +4972,13 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tVariantListItem* pItem = taosArrayGet(pQuerySql->pSortOrder, 0); + tVariantListItem* pItem = taosArrayGet(pQuerySqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId; return TSDB_CODE_SUCCESS; } - tVariantListItem* pItem = taosArrayGet(pQuerySql->pSortOrder, 0); + tVariantListItem* pItem = taosArrayGet(pQuerySqlNode->pSortOrder, 0); pQueryInfo->order.order = pItem->sortOrder; } @@ -4788,11 +5009,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg17 = "invalid column name"; const char* msg18 = "primary timestamp column cannot be dropped"; const char* msg19 = "invalid new tag name"; + const char* msg20 = "table is not super table"; int32_t code = TSDB_CODE_SUCCESS; SSqlCmd* pCmd = &pSql->cmd; - SAlterTableSQL* pAlterSQL = pInfo->pAlterInfo; + SAlterTableInfo* pAlterSQL = pInfo->pAlterInfo; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, DEFAULT_TABLE_INDEX); @@ -4813,6 +5035,10 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + if (pAlterSQL->tableType == TSDB_SUPER_TABLE && !(UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo))) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg20); + } + if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN || pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { @@ -4869,7 +5095,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[128] = {0}; strncpy(name1, pItem->pVar.pz, pItem->pVar.nLen); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) { SArray* pVarList = pAlterSQL->varList; @@ -4905,14 +5131,14 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name[TSDB_COL_NAME_LEN] = {0}; strncpy(name, pItem->pVar.pz, pItem->pVar.nLen); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypes[TSDB_DATA_TYPE_INT].bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); pItem = taosArrayGet(pVarList, 1); memset(name, 0, tListLen(name)); strncpy(name, pItem->pVar.pz, pItem->pVar.nLen); - f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypes[TSDB_DATA_TYPE_INT].bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { // Note: update can only be applied to table not super table. @@ -4987,7 +5213,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t len = 0; if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) { - len = tDataTypeDesc[pTagsSchema->type].nSize; + len = tDataTypes[pTagsSchema->type].bytes; } else { len = varDataTLen(pUpdateMsg->data + schemaLen); } @@ -5034,7 +5260,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { char name1[TSDB_COL_NAME_LEN] = {0}; tstrncpy(name1, pItem->pVar.pz, sizeof(name1)); - TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize); + TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypes[TSDB_DATA_TYPE_INT].bytes); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); } @@ -5054,7 +5280,7 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { int32_t functId = tscSqlExprGet(pQueryInfo, i)->functionId; - if (!IS_STREAM_QUERY_VALID(aAggs[functId].nStatus)) { + if (!IS_STREAM_QUERY_VALID(aAggs[functId].status)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } } @@ -5077,7 +5303,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu bool hasSelectivity = false; for (int32_t j = 0; j < size; ++j) { SSqlExpr* pEx = tscSqlExprGet(pQueryInfo, j); - if ((aAggs[pEx->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { + if ((aAggs[pEx->functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) { hasSelectivity = true; break; } @@ -5127,8 +5353,10 @@ int32_t validateEp(char* ep) { return TSDB_CODE_SUCCESS; } -int32_t validateDNodeConfig(tDCLSQL* pOptions) { - if (pOptions->nTokens < 2 || pOptions->nTokens > 3) { +int32_t validateDNodeConfig(SMiscInfo* pOptions) { + int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); + + if (numOfToken < 2 || numOfToken > 3) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -5146,9 +5374,9 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { {"cqDebugFlag", 11}, }; - SStrToken* pOptionToken = &pOptions->a[1]; + SStrToken* pOptionToken = taosArrayGet(pOptions->a, 1); - if (pOptions->nTokens == 2) { + if (numOfToken == 2) { // reset log and reset query cache does not need value for (int32_t i = 0; i < tokenLogEnd; ++i) { const SDNodeDynConfOption* pOption = &cfgOptions[i]; @@ -5158,7 +5386,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { } } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SStrToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = taosArrayGet(pOptions->a, 2); int32_t vnodeId = 0; int32_t dnodeId = 0; strdequote(pValToken->z); @@ -5169,14 +5397,14 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_SUCCESS; } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SStrToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = taosArrayGet(pOptions->a, 2); int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid } return TSDB_CODE_SUCCESS; } else { - SStrToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = taosArrayGet(pOptions->a, 2); int32_t val = strtol(pValToken->z, NULL, 10); if (val < 0 || val > 256) { @@ -5187,8 +5415,8 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { for (int32_t i = tokenDebugFlag; i < tokenDebugFlagEnd; ++i) { const SDNodeDynConfOption* pOption = &cfgOptions[i]; + // options is valid if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { - /* options is valid */ return TSDB_CODE_SUCCESS; } } @@ -5197,37 +5425,39 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_TSC_INVALID_SQL; } -int32_t validateLocalConfig(tDCLSQL* pOptions) { - if (pOptions->nTokens < 1 || pOptions->nTokens > 2) { +int32_t validateLocalConfig(SMiscInfo* pOptions) { + int32_t numOfToken = (int32_t) taosArrayGetSize(pOptions->a); + if (numOfToken < 1 || numOfToken > 2) { return TSDB_CODE_TSC_INVALID_SQL; } SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - SStrToken* pOptionToken = &pOptions->a[0]; - if (pOptions->nTokens == 1) { + SStrToken* pOptionToken = taosArrayGet(pOptions->a, 0); + + if (numOfToken == 1) { // reset log does not need value for (int32_t i = 0; i < 1; ++i) { SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { + if ((pOption->len == pOptionToken->n) && + (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { return TSDB_CODE_SUCCESS; } } } else { - SStrToken* pValToken = &pOptions->a[1]; + SStrToken* pValToken = taosArrayGet(pOptions->a, 1); int32_t val = strtol(pValToken->z, NULL, 10); - if (val < 131 || val > 199) { - // options value is out of valid range + if (!validateDebugFlag(val)) { return TSDB_CODE_TSC_INVALID_SQL; } for (int32_t i = 1; i < tListLen(LOCAL_DYNAMIC_CFG_OPTIONS); ++i) { SDNodeDynConfOption* pOption = &LOCAL_DYNAMIC_CFG_OPTIONS[i]; - if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { - // options is valid + if ((pOption->len == pOptionToken->n) + && (strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0)) { return TSDB_CODE_SUCCESS; } } @@ -5250,6 +5480,7 @@ int32_t validateColumnName(char* name) { if (token.type == TK_STRING) { strdequote(token.z); + strntolower(token.z, token.z, token.n); token.n = (uint32_t)strtrim(token.z); int32_t k = tSQLGetToken(token.z, &token.type); @@ -5275,7 +5506,7 @@ bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo) { return (pQueryInfo->window.skey == pQueryInfo->window.ekey) && (pQueryInfo->window.skey != 0); } -int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* pQuerySql, SSqlObj* pSql) { +int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySqlNode* pQuerySqlNode, SSqlObj* pSql) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); const char* msg0 = "soffset/offset can not be less than 0"; @@ -5283,9 +5514,9 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn const char* msg2 = "slimit/soffset can not apply to projection query"; // handle the limit offset value, validate the limit - pQueryInfo->limit = pQuerySql->limit; + pQueryInfo->limit = pQuerySqlNode->limit; pQueryInfo->clauseLimit = pQueryInfo->limit.limit; - pQueryInfo->slimit = pQuerySql->slimit; + pQueryInfo->slimit = pQuerySqlNode->slimit; tscDebug("%p limit:%" PRId64 ", offset:%" PRId64 " slimit:%" PRId64 ", soffset:%" PRId64, pSql, pQueryInfo->limit.limit, pQueryInfo->limit.offset, pQueryInfo->slimit.limit, pQueryInfo->slimit.offset); @@ -5368,7 +5599,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn return TSDB_CODE_SUCCESS; } -static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { +static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) { const char* msg = "invalid number of options"; pMsg->daysToKeep = htonl(-1); @@ -5406,7 +5637,7 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* p return TSDB_CODE_SUCCESS; } -static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) { +static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDbInfo) { const char* msg = "invalid time precision"; pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default @@ -5430,7 +5661,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo return TSDB_CODE_SUCCESS; } -static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { +static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDbInfo* pCreateDb) { pMsg->maxTables = htonl(-1); // max tables can not be set anymore pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); @@ -5446,9 +5677,11 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->ignoreExist = pCreateDb->ignoreExists; pMsg->update = pCreateDb->update; pMsg->cacheLastRow = pCreateDb->cachelast; + pMsg->dbType = pCreateDb->dbType; + pMsg->partitions = htons(pCreateDb->partitions); } -int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { +int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDbInfo* pCreateDbSql) { SCreateDbMsg* pMsg = (SCreateDbMsg *)(pCmd->payload); setCreateDBOption(pMsg, pCreateDbSql); @@ -5527,7 +5760,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->colIndex); SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pColIndex->colIndex}; - tscAddSpecialColumnForSelect(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL); int32_t numOfFields = tscNumOfFields(pQueryInfo); SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfFields - 1); @@ -5691,7 +5924,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) continue; } - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { numOfSelectivity++; } else { numOfAggregation++; @@ -5723,7 +5956,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd) for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int16_t functionId = pExpr->functionId; - if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) { + if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) == 0) { continue; } @@ -5766,7 +5999,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SSchema s = tGetTableNameColumnSchema(); + SSchema s = *tGetTbnameColumnSchema(); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); int16_t bytes = 0; int16_t type = 0; @@ -5808,8 +6041,8 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo SColumnList ids = getColumnList(1, 0, pColIndex->colIndex); insertResultField(pQueryInfo, (int32_t)size, &ids, bytes, (int8_t)type, name, pExpr); } else { - // if this query is "group by" normal column, interval is not allowed - if (pQueryInfo->interval.interval > 0) { + // if this query is "group by" normal column, time window query is not allowed + if (isTimeWindowQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -5871,7 +6104,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || pQueryInfo->interval.interval > 0) { + if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || isTimeWindowQuery(pQueryInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } else { return TSDB_CODE_SUCCESS; @@ -5910,7 +6143,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { } } - if (IS_MULTIOUTPUT(aAggs[functId].nStatus) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM && + if (IS_MULTIOUTPUT(aAggs[functId].status) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM && functId != TSDB_FUNC_TAGPRJ && functId != TSDB_FUNC_PRJ) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5942,17 +6175,20 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return checkUpdateTagPrjFunctions(pQueryInfo, pCmd); } } -int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySqlNode* pQuerySqlNode) { const char* msg1 = "only one expression allowed"; const char* msg2 = "invalid expression in select clause"; const char* msg3 = "invalid function"; - tSQLExprList* pExprList = pQuerySql->pSelection; - if (pExprList->nExpr != 1) { + SArray* pExprList = pQuerySqlNode->pSelectList; + size_t size = taosArrayGetSize(pExprList); + if (size != 1) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } + bool server_status = false; - tSQLExpr* pExpr = pExprList->a[0].pNode; + tSqlExprItem* pExprItem = taosArrayGet(pExprList, 0); + tSqlExpr* pExpr = pExprItem->pNode; if (pExpr->operand.z == NULL) { //handle 'select 1' if (pExpr->token.n == 1 && 0 == strncasecmp(pExpr->token.z, "1", 1)) { @@ -5973,8 +6209,8 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ index = 2; } else { for (int32_t i = 0; i < tListLen(functionsInfo); ++i) { - if (strncasecmp(functionsInfo[i].name, pExpr->operand.z, functionsInfo[i].len) == 0 && - functionsInfo[i].len == pExpr->operand.n) { + if (strncasecmp(functionsInfo[i].name, pExpr->token.z, functionsInfo[i].len) == 0 && + functionsInfo[i].len == pExpr->token.n) { index = i; break; } @@ -5997,9 +6233,10 @@ int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ SColumnIndex ind = {0}; SSqlExpr* pExpr1 = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TAG_DUMMY, &ind, TSDB_DATA_TYPE_INT, - tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, getNewResColId(pQueryInfo), tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize, false); - - const char* name = (pExprList->a[0].aliasName != NULL)? pExprList->a[0].aliasName:functionsInfo[index].name; + tDataTypes[TSDB_DATA_TYPE_INT].bytes, getNewResColId(pQueryInfo), tDataTypes[TSDB_DATA_TYPE_INT].bytes, false); + + tSqlExprItem* item = taosArrayGet(pExprList, 0); + const char* name = (item->aliasName != NULL)? item->aliasName:functionsInfo[index].name; tstrncpy(pExpr1->aliasName, name, tListLen(pExpr1->aliasName)); return TSDB_CODE_SUCCESS; @@ -6076,6 +6313,15 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } + val = (int16_t)htons(pCreate->partitions); + if (val != -1 && + (val < TSDB_MIN_DB_PARTITON_OPTION || val > TSDB_MAX_DB_PARTITON_OPTION)) { + snprintf(msg, tListLen(msg), "invalid topic option partition: %d valid range: [%d, %d]", val, + TSDB_MIN_DB_PARTITON_OPTION, TSDB_MAX_DB_PARTITON_OPTION); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); + } + + return TSDB_CODE_SUCCESS; } @@ -6100,7 +6346,7 @@ void tscPrintSelectClause(SSqlObj* pSql, int32_t subClauseIndex) { char tmpBuf[1024] = {0}; int32_t tmpLen = 0; tmpLen = - sprintf(tmpBuf, "%s(uid:%" PRId64 ", %d)", aAggs[pExpr->functionId].aName, pExpr->uid, pExpr->colInfo.colId); + sprintf(tmpBuf, "%s(uid:%" PRId64 ", %d)", aAggs[pExpr->functionId].name, pExpr->uid, pExpr->colInfo.colId); if (tmpLen + offset >= totalBufSize - 1) break; @@ -6125,7 +6371,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo; + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; SArray* pFieldList = pCreateTable->colInfo.pColumns; SArray* pTagList = pCreateTable->colInfo.pTagColumns; @@ -6180,7 +6426,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; - SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo; + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); // two table: the first one is for current table, and the secondary is for the super table. @@ -6209,18 +6455,16 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } // get table meta from mnode - tstrncpy(pCreateTableInfo->tagdata.name, pStableMetaInfo->name, tListLen(pCreateTableInfo->tagdata.name)); - SArray* pList = pCreateTableInfo->pTagVals; + code = tNameExtractFullName(&pStableMetaInfo->name, pCreateTableInfo->tagdata.name); + SArray* pValList = pCreateTableInfo->pTagVals; code = tscGetTableMeta(pSql, pStableMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; } - size_t size = taosArrayGetSize(pList); - if (tscGetNumOfTags(pStableMetaInfo->pTableMeta) != size) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); - } + size_t valSize = taosArrayGetSize(pValList); + // too long tag values will return invalid sql, not be truncated automatically SSchema *pTagSchema = tscGetTableTagSchema(pStableMetaInfo->pTableMeta); @@ -6231,36 +6475,115 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } + + SArray* pNameList = NULL; + size_t nameSize = 0; + int32_t schemaSize = tscGetNumOfTags(pStableMetaInfo->pTableMeta); int32_t ret = TSDB_CODE_SUCCESS; - for (int32_t i = 0; i < size; ++i) { - SSchema* pSchema = &pTagSchema[i]; - tVariantListItem* pItem = taosArrayGet(pList, i); - char tagVal[TSDB_MAX_TAGS_LEN]; - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - if (pItem->pVar.nLen > pSchema->bytes) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); - } + if (pCreateTableInfo->pTagNames) { + pNameList = pCreateTableInfo->pTagNames; + nameSize = taosArrayGetSize(pNameList); + + if (valSize != nameSize) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + } + + if (schemaSize < valSize) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); + bool findColumnIndex = false; + + for (int32_t i = 0; i < nameSize; ++i) { + SStrToken* sToken = taosArrayGet(pNameList, i); + if (TK_STRING == sToken->type) { + tscDequoteAndTrimToken(sToken); + } + + tVariantListItem* pItem = taosArrayGet(pValList, i); + + findColumnIndex = false; + + // todo speedup by using hash list + for (int32_t t = 0; t < schemaSize; ++t) { + if (strncmp(sToken->z, pTagSchema[t].name, sToken->n) == 0 && strlen(pTagSchema[t].name) == sToken->n) { + SSchema* pSchema = &pTagSchema[t]; - // check again after the convert since it may be converted from binary to nchar. - if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { - int16_t len = varDataTLen(tagVal); - if (len > pSchema->bytes) { + char tagVal[TSDB_MAX_TAGS_LEN]; + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + if (pItem->pVar.nLen > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); + + // check again after the convert since it may be converted from binary to nchar. + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + int16_t len = varDataTLen(tagVal); + if (len > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + if (ret != TSDB_CODE_SUCCESS) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); + + findColumnIndex = true; + break; + } + } + + if (!findColumnIndex) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + return tscInvalidSQLErrMsg(pCmd->payload, "invalid tag name", sToken->z); } } - - if (ret != TSDB_CODE_SUCCESS) { + } else { + if (schemaSize != valSize) { tdDestroyKVRowBuilder(&kvRowBuilder); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); + for (int32_t i = 0; i < valSize; ++i) { + SSchema* pSchema = &pTagSchema[i]; + tVariantListItem* pItem = taosArrayGet(pValList, i); + + char tagVal[TSDB_MAX_TAGS_LEN]; + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + if (pItem->pVar.nLen > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); + + // check again after the convert since it may be converted from binary to nchar. + if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { + int16_t len = varDataTLen(tagVal); + if (len > pSchema->bytes) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + if (ret != TSDB_CODE_SUCCESS) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + + tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); + } } SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); @@ -6289,7 +6612,11 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return ret; } - pCreateTableInfo->fullname = strndup(pTableMetaInfo->name, TSDB_TABLE_FNAME_LEN); + pCreateTableInfo->fullname = calloc(1, tNameLen(&pTableMetaInfo->name) + 1); + ret = tNameExtractFullName(&pTableMetaInfo->name, pCreateTableInfo->fullname); + if (ret != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } } return TSDB_CODE_SUCCESS; @@ -6308,24 +6635,24 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); assert(pQueryInfo->numOfTables == 1); - SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo; + SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // if sql specifies db, use it, otherwise use default db SStrToken* pName = &(pCreateTable->name); - SQuerySQL* pQuerySql = pCreateTable->pSelect; + SQuerySqlNode* pQuerySqlNode = pCreateTable->pSelect; if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - SArray* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from; - if (pSrcMeterName == NULL || taosArrayGetSize(pSrcMeterName) == 0) { + SFromInfo* pFromInfo = pInfo->pCreateTableInfo->pSelect->from; + if (pFromInfo == NULL || taosArrayGetSize(pFromInfo->tableList) == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - tVariantListItem* p1 = taosArrayGet(pSrcMeterName, 0); - SStrToken srcToken = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING}; + STableNamePair* p1 = taosArrayGet(pFromInfo->tableList, 0); + SStrToken srcToken = {.z = p1->name.z, .n = p1->name.n, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -6341,23 +6668,22 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false, false) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(&pSql->cmd, 0, pQuerySqlNode->pSelectList, isSTable, false, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - if (pQuerySql->pWhere != NULL) { // query condition in stream computing - if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) { + if (pQuerySqlNode->pWhere != NULL) { // query condition in stream computing + if (parseWhereClause(pQueryInfo, &pQuerySqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } } // set interval value - if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pSql, pQueryInfo, pQuerySqlNode) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - if ((pQueryInfo->interval.interval > 0) && - (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { + if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -6371,7 +6697,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return code; } - if (pQuerySql->selectToken.n > TSDB_MAX_SAVED_SQL_LEN) { + if (pQuerySqlNode->sqlstr.n > TSDB_MAX_SAVED_SQL_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } @@ -6389,12 +6715,12 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { * check if fill operation is available, the fill operation is parsed and executed during query execution, * not here. */ - if (pQuerySql->fillType != NULL) { + if (pQuerySqlNode->fillType != NULL) { if (pQueryInfo->interval.interval == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - tVariantListItem* pItem = taosArrayGet(pQuerySql->fillType, 0); + tVariantListItem* pItem = taosArrayGet(pQuerySqlNode->fillType, 0); if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) { if (!((strncmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) || (strncmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4))) { @@ -6443,17 +6769,17 @@ static int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) { return TSDB_CODE_SUCCESS; } -int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { - assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0)); +int32_t doValidateSqlNode(SSqlObj* pSql, SQuerySqlNode* pQuerySqlNode, int32_t index) { + assert(pQuerySqlNode != NULL && (pQuerySqlNode->from == NULL || taosArrayGetSize(pQuerySqlNode->from->tableList) > 0)); - const char* msg0 = "invalid table name"; + const char* msg0 = "invalid table name"; const char* msg1 = "point interpolation query needs timestamp"; const char* msg2 = "fill only available for interval query"; const char* msg3 = "start(end) time of query range required or time range too large"; - const char* msg4 = "illegal number of tables in from clause"; const char* msg5 = "too many columns in selection clause"; const char* msg6 = "too many tables in from clause"; const char* msg7 = "invalid table alias name"; + const char* msg8 = "alias name too long"; int32_t code = TSDB_CODE_SUCCESS; @@ -6468,7 +6794,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { assert(pCmd->clauseIndex == index); // too many result columns not support order by in query - if (pQuerySql->pSelection->nExpr > TSDB_MAX_COLUMNS) { + if (taosArrayGetSize(pQuerySqlNode->pSelectList) > TSDB_MAX_COLUMNS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } @@ -6479,77 +6805,74 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { * select client_version(); * select server_state(); */ - if (pQuerySql->from == NULL) { - assert(pQuerySql->fillType == NULL && pQuerySql->pGroupby == NULL && pQuerySql->pWhere == NULL && - pQuerySql->pSortOrder == NULL); - return doLocalQueryProcess(pCmd, pQueryInfo, pQuerySql); + if (pQuerySqlNode->from == NULL) { + assert(pQuerySqlNode->fillType == NULL && pQuerySqlNode->pGroupby == NULL && pQuerySqlNode->pWhere == NULL && + pQuerySqlNode->pSortOrder == NULL); + return doLocalQueryProcess(pCmd, pQueryInfo, pQuerySqlNode); } - size_t fromSize = taosArrayGetSize(pQuerySql->from); + size_t fromSize = taosArrayGetSize(pQuerySqlNode->from->tableList); if (fromSize > TSDB_MAX_JOIN_TABLE_NUM * 2) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } pQueryInfo->command = TSDB_SQL_SELECT; - if (fromSize > 4) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); - } - // set all query tables, which are maybe more than one. - for (int32_t i = 0; i < fromSize; ) { - tVariantListItem* item = taosArrayGet(pQuerySql->from, i); - tVariant* pTableItem = &item->pVar; + for (int32_t i = 0; i < fromSize; ++i) { + STableNamePair* item = taosArrayGet(pQuerySqlNode->from->tableList, i); + SStrToken* pTableItem = &item->name; - if (pTableItem->nType != TSDB_DATA_TYPE_BINARY) { + if (pTableItem->type != TSDB_DATA_TYPE_BINARY) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } - pTableItem->nLen = strdequote(pTableItem->pz); + tscDequoteAndTrimToken(pTableItem); - SStrToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; + SStrToken tableName = {.z = pTableItem->z, .n = pTableItem->n, .type = TK_STRING}; if (tscValidateName(&tableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } - if (pQueryInfo->numOfTables <= i/2) { // more than one table + if (pQueryInfo->numOfTables <= i) { // more than one table tscAddEmptyMetaInfo(pQueryInfo); } - STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2); - - SStrToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; - code = tscSetTableFullName(pTableMetaInfo1, &t, pSql); + STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i); + code = tscSetTableFullName(pTableMetaInfo1, pTableItem, pSql); if (code != TSDB_CODE_SUCCESS) { return code; } - tVariantListItem* p1 = taosArrayGet(pQuerySql->from, i + 1); - if (p1->pVar.nType != TSDB_DATA_TYPE_BINARY) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); - } + SStrToken* aliasName = &item->aliasName; + if (TPARSER_HAS_TOKEN(*aliasName)) { + if (aliasName->type != TSDB_DATA_TYPE_BINARY) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + } - SStrToken aliasName = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING}; - if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); - } + tscDequoteAndTrimToken(aliasName); + + SStrToken aliasName1 = {.z = aliasName->z, .n = aliasName->n, .type = TK_STRING}; + if (tscValidateName(&aliasName1) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); + } + + if (aliasName1.n >= TSDB_TABLE_NAME_LEN) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); + } - // has no table alias name - if (memcmp(pTableItem->pz, p1->pVar.pz, p1->pVar.nLen) == 0) { - extractTableName(pTableMetaInfo1->name, pTableMetaInfo1->aliasName); + strncpy(pTableMetaInfo1->aliasName, aliasName1.z, aliasName1.n); } else { - tstrncpy(pTableMetaInfo1->aliasName, p1->pVar.pz, sizeof(pTableMetaInfo1->aliasName)); + strncpy(pTableMetaInfo1->aliasName, tNameGetTableName(&pTableMetaInfo1->name), tListLen(pTableMetaInfo1->aliasName)); } code = tscGetTableMeta(pSql, pTableMetaInfo1); if (code != TSDB_CODE_SUCCESS) { return code; } - - i += 2; } - assert(pQueryInfo->numOfTables == taosArrayGetSize(pQuerySql->from) / 2); + assert(pQueryInfo->numOfTables == taosArrayGetSize(pQuerySqlNode->from->tableList)); bool isSTable = false; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { @@ -6565,52 +6888,56 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } // parse the group by clause in the first place - if (parseGroupbyClause(pQueryInfo, pQuerySql->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { + if (parseGroupbyClause(pQueryInfo, pQuerySqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } // set where info STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - if (pQuerySql->pWhere != NULL) { - if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) { + if (pQuerySqlNode->pWhere != NULL) { + if (parseWhereClause(pQueryInfo, &pQuerySqlNode->pWhere, pSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - pQuerySql->pWhere = NULL; + pQuerySqlNode->pWhere = NULL; if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { pQueryInfo->window.skey = pQueryInfo->window.skey / 1000; pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000; } } else { // set the time rang - if (taosArrayGetSize(pQuerySql->from) > 2) { // it is a join query, no wher clause is not allowed. + if (taosArrayGetSize(pQuerySqlNode->from->tableList) > 1) { // it is a join query, no where clause is not allowed. return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); } } - int32_t joinQuery = (pQuerySql->from != NULL && taosArrayGetSize(pQuerySql->from) > 2); + int32_t joinQuery = (pQuerySqlNode->from != NULL && taosArrayGetSize(pQuerySqlNode->from->tableList) > 1); + int32_t timeWindowQuery = + (TPARSER_HAS_TOKEN(pQuerySqlNode->interval.interval) || TPARSER_HAS_TOKEN(pQuerySqlNode->sessionVal.gap)); - int32_t intervalQuery = !(pQuerySql->interval.type == 0 || pQuerySql->interval.n == 0); - - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery, intervalQuery) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(pCmd, index, pQuerySqlNode->pSelectList, isSTable, joinQuery, timeWindowQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } // set order by info - if (parseOrderbyClause(pCmd, pQueryInfo, pQuerySql, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) { + if (parseOrderbyClause(pCmd, pQueryInfo, pQuerySqlNode, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } // set interval value - if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pSql, pQueryInfo, pQuerySqlNode) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { - if ((pQueryInfo->interval.interval > 0) && + if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return TSDB_CODE_TSC_INVALID_SQL; } } + if (parseSessionClause(pCmd, pQueryInfo, pQuerySqlNode) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + // no result due to invalid query time range if (pQueryInfo->window.skey > pQueryInfo->window.ekey) { pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; @@ -6624,13 +6951,12 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { // in case of join query, time range is required. if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); - if (timeRange == 0 && pQueryInfo->window.skey == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - if ((code = parseLimitClause(pCmd, pQueryInfo, index, pQuerySql, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = parseLimitClause(pCmd, pQueryInfo, index, pQuerySqlNode, pSql)) != TSDB_CODE_SUCCESS) { return code; } @@ -6638,9 +6964,10 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return code; } + updateLastScanOrderIfNeeded(pQueryInfo); tscFieldInfoUpdateOffset(pQueryInfo); - if (pQuerySql->fillType != NULL) { + if (pQuerySqlNode->fillType != NULL) { if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -6653,7 +6980,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return code; } - if ((code = parseFillClause(pCmd, pQueryInfo, pQuerySql)) != TSDB_CODE_SUCCESS) { + if ((code = parseFillClause(pCmd, pQueryInfo, pQuerySqlNode)) != TSDB_CODE_SUCCESS) { return code; } } @@ -6661,7 +6988,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid) { +int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSqlExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; @@ -6679,25 +7006,25 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS } } - if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->nSQLOptr == 0) { + if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->tokenId == 0) { *pExpr = calloc(1, sizeof(tExprNode)); return TSDB_CODE_SUCCESS; } if (pSqlExpr->pLeft == NULL) { - if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) { + if (pSqlExpr->type == SQL_NODE_VALUE) { *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_VALUE; (*pExpr)->pVal = calloc(1, sizeof(tVariant)); - tVariantAssign((*pExpr)->pVal, &pSqlExpr->val); + tVariantAssign((*pExpr)->pVal, &pSqlExpr->value); return TSDB_CODE_SUCCESS; - } else if (pSqlExpr->nSQLOptr >= TK_COUNT && pSqlExpr->nSQLOptr <= TK_AVG_IRATE) { + } else if (pSqlExpr->type == SQL_NODE_SQLFUNCTION) { // arithmetic expression on the results of aggregation functions *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_COL; (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); + strncpy((*pExpr)->pSchema->name, pSqlExpr->token.z, pSqlExpr->token.n); // set the input column data byte and type. size_t size = taosArrayGetSize(pQueryInfo->exprList); @@ -6717,7 +7044,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS break; } } - } else if (pSqlExpr->nSQLOptr == TK_ID) { // column name, normal column arithmetic expression + } else if (pSqlExpr->type == SQL_NODE_TABLE_COLUMN) { // column name, normal column arithmetic expression SColumnIndex index = COLUMN_INDEX_INITIALIZER; int32_t ret = getColumnIndexByName(pCmd, &pSqlExpr->colInfo, pQueryInfo, &index); if (ret != TSDB_CODE_SUCCESS) { @@ -6757,7 +7084,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SStrToken t = {.type = pSqlExpr->nSQLOptr}; + SStrToken t = {.type = pSqlExpr->tokenId}; (*pExpr)->_node.optr = convertOptr(&t); assert((*pExpr)->_node.optr != 0); @@ -6797,3 +7124,4 @@ bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { return false; } + diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 123f0fd2227d56f2b67dd7040afe8dea22c681ba..67352ca71cab03bb9cd2f6b920b97abebbe1420a 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -66,68 +66,6 @@ STableComInfo tscGetTableInfo(const STableMeta* pTableMeta) { return pTableMeta->tableInfo; } -static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { - int32_t rowLen = 0; - - for (int32_t i = 0; i < numOfCols; ++i) { - // 1. valid types - if (!isValidDataType(pSchema[i].type)) { - return false; - } - - // 2. valid length for each type - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { - if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { - return false; - } - } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { - if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { - return false; - } - } else { - if (pSchema[i].bytes != tDataTypeDesc[pSchema[i].type].nSize) { - return false; - } - } - - // 3. valid column names - for (int32_t j = i + 1; j < numOfCols; ++j) { - if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) { - return false; - } - } - - rowLen += pSchema[i].bytes; - } - - return rowLen <= maxLen; -} - -bool isValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) { - if (!VALIDNUMOFCOLS(numOfCols)) { - return false; - } - - if (!VALIDNUMOFTAGS(numOfTags)) { - return false; - } - - /* first column must be the timestamp, which is a primary key */ - if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) { - return false; - } - - if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) { - return false; - } - - if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) { - return false; - } - - return true; -} - SSchema* tscGetTableColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) { assert(pTableMeta != NULL); @@ -168,6 +106,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; + tstrncpy(pTableMeta->sTableName, pTableMetaMsg->sTableName, TSDB_TABLE_FNAME_LEN); memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 15dd77faebdb4167a9d13f21b2606a0946199b8e..7085318e350a4236c83960368bd1cb813a7df4df 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -25,8 +25,6 @@ #include "ttimer.h" #include "tlockfree.h" -///SRpcCorEpSet tscMgmtEpSet; - int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; int (*tscProcessMsgRsp[TSDB_SQL_MAX])(SSqlObj *pSql); @@ -159,13 +157,16 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SRpcEpSet *epSet = &pRsp->epSet; if (epSet->numOfEps > 0) { tscEpSetHtons(epSet); - if (!tscEpSetIsEqual(&pSql->pTscObj->tscCorMgmtEpSet->epSet, epSet)) { - tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); - for (int8_t i = 0; i < epSet->numOfEps; i++) { - tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); - } - tscUpdateMgmtEpSet(pSql, epSet); - } + + //SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + //if (!tscEpSetIsEqual(&pCorEpSet->epSet, epSet)) { + // tscTrace("%p updating epset: numOfEps: %d, inUse: %d", pSql, epSet->numOfEps, epSet->inUse); + // for (int8_t i = 0; i < epSet->numOfEps; i++) { + // tscTrace("endpoint %d: fqdn=%s, port=%d", i, epSet->fqdn[i], epSet->port[i]); + // } + //} + //concurrency problem, update mgmt epset anyway + tscUpdateMgmtEpSet(pSql, epSet); } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -272,7 +273,8 @@ int tscSendMsgToServer(SSqlObj *pSql) { .code = 0 }; - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); + + rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid); return TSDB_CODE_SUCCESS; } @@ -294,8 +296,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pObj->signature != pObj) { tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); - taosRemoveRef(tscObjRef, pSql->self); - taosReleaseRef(tscObjRef, pSql->self); + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); return; } @@ -305,13 +307,13 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - taosRemoveRef(tscObjRef, pSql->self); - taosReleaseRef(tscObjRef, pSql->self); + taosRemoveRef(tscObjRef, handle); + taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); return; } - if (pEpSet) { // todo update this + if (pEpSet) { if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { if (pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); @@ -328,7 +330,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { pSql->cmd.submitSchema = 1; } - if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || @@ -341,17 +343,18 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pSql->retry > pSql->maxRetry) { tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); } else { - // wait for a little bit moment and then retry, todo do not sleep in rpc callback thread + // wait for a little bit moment and then retry + // todo do not sleep in rpc callback thread, add this process into queueu to process if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { int32_t duration = getWaitingTimeInterval(pSql->retry); taosMsleep(duration); } + pSql->retryReason = rpcMsg->code; rpcMsg->code = tscRenewTableMeta(pSql, 0); - // if there is an error occurring, proceed to the following error handling procedure. if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosReleaseRef(tscObjRef, pSql->self); + taosReleaseRef(tscObjRef, handle); rpcFreeCont(rpcMsg->pCont); return; } @@ -415,17 +418,22 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { bool shouldFree = tscShouldBeFreed(pSql); if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + if (rpcMsg->code != TSDB_CODE_SUCCESS) { + pRes->code = rpcMsg->code; + } rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code; (*pSql->fp)(pSql->param, pSql, rpcMsg->code); } - taosReleaseRef(tscObjRef, pSql->self); + if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - taosRemoveRef(tscObjRef, pSql->self); + taosRemoveRef(tscObjRef, handle); tscDebug("%p sqlObj is automatically freed", pSql); } + taosReleaseRef(tscObjRef, handle); + rpcFreeCont(rpcMsg->pCont); } @@ -446,7 +454,7 @@ int doProcessSql(SSqlObj *pSql) { if (pRes->code != TSDB_CODE_SUCCESS) { tscAsyncResultOnError(pSql); - return pRes->code; + return TSDB_CODE_SUCCESS; } int32_t code = tscSendMsgToServer(pSql); @@ -455,23 +463,27 @@ int doProcessSql(SSqlObj *pSql) { if (code != TSDB_CODE_SUCCESS) { pRes->code = code; tscAsyncResultOnError(pSql); - return code; + return TSDB_CODE_SUCCESS; } return TSDB_CODE_SUCCESS; } int tscProcessSql(SSqlObj *pSql) { - char *name = NULL; + char name[TSDB_TABLE_FNAME_LEN] = {0}; + SSqlCmd *pCmd = &pSql->cmd; - + uint32_t type = 0; + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = NULL; - uint32_t type = 0; if (pQueryInfo != NULL) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - name = (pTableMetaInfo != NULL)? pTableMetaInfo->name:NULL; + if (pTableMetaInfo != NULL) { + tNameExtractFullName(&pTableMetaInfo->name, name); + } + type = pQueryInfo->type; // while numOfTables equals to 0, it must be Heartbeat @@ -484,9 +496,7 @@ int tscProcessSql(SSqlObj *pSql) { pSql->res.code = TSDB_CODE_TSC_APP_ERROR; return pSql->res.code; } - } else if (pCmd->command < TSDB_SQL_LOCAL) { - //pSql->epSet = tscMgmtEpSet; - } else { // local handler + } else if (pCmd->command >= TSDB_SQL_LOCAL) { return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -498,7 +508,7 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); pRetrieveMsg->free = htons(pQueryInfo->type); - pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle); + pRetrieveMsg->qId = htobe64(pSql->res.qId); // todo valid the vgroupId at the client side STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); @@ -510,7 +520,7 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); - tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d, qhandle:%" PRIX64, pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex, pSql->res.qId); } else { int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); assert(vgIndex >= 0 && vgIndex < numOfVgroups); @@ -518,12 +528,12 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); pRetrieveMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId); - tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pTableIdList->vgInfo.vgId, vgIndex); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d, qId:%" PRIu64, pSql, pTableIdList->vgInfo.vgId, vgIndex, pSql->res.qId); } } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pRetrieveMsg->header.vgId = htonl(pTableMeta->vgId); - tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgId); + tscDebug("%p build fetch msg from only one vgroup, vgId:%d, qId:%" PRIu64, pSql, pTableMeta->vgId, pSql->res.qId); } pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); @@ -600,10 +610,10 @@ static int32_t tscEstimateQueryMsgSize(SSqlObj *pSql, int32_t clauseIndex) { } return MIN_QUERY_MSG_PKT_SIZE + minMsgSize() + sizeof(SQueryTableMsg) + srcColListSize + exprSize + tsBufSize + - tableSerialize + sqlLen + 4096; + tableSerialize + sqlLen + 4096 + pQueryInfo->bufLen; } -static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg) { +static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char *pMsg, int32_t *succeed) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); TSKEY dfltKey = htobe64(pQueryMsg->window.skey); @@ -616,9 +626,14 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char assert(index >= 0); SVgroupInfo* pVgroupInfo = NULL; - if (pTableMetaInfo->vgroupList->numOfVgroups > 0) { + if (pTableMetaInfo->vgroupList && pTableMetaInfo->vgroupList->numOfVgroups > 0) { assert(index < pTableMetaInfo->vgroupList->numOfVgroups); pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; + } else { + tscError("%p No vgroup info found", pSql); + + *succeed = 0; + return pMsg; } vgId = pVgroupInfo->vgId; @@ -633,7 +648,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } pSql->epSet.inUse = rand()%pSql->epSet.numOfEps; - pQueryMsg->head.vgId = htonl(vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; @@ -648,8 +662,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); assert(index >= 0 && index < numOfVgroups); - tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, numOfVgroups); - SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info @@ -658,7 +670,10 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char int32_t numOfTables = (int32_t)taosArrayGetSize(pTableIdList->itemList); pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables - + + tscDebug("%p query on stable, vgId:%d, numOfTables:%d, vgIndex:%d, numOfVgroups:%d", pSql, + pTableIdList->vgInfo.vgId, numOfTables, index, numOfVgroups); + // serialize each table id info for(int32_t i = 0; i < numOfTables; ++i) { STableIdInfo* pItem = taosArrayGet(pTableIdList->itemList, i); @@ -670,10 +685,11 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char pMsg += sizeof(STableIdInfo); } } - - tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, - pTableMeta->id.tid, pTableMeta->id.uid); - + + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + + tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), n, pTableMeta->id.tid, pTableMeta->id.uid); return pMsg; } @@ -692,7 +708,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); - if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) { tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", pSql, (uint64_t)numOfSrcCols, tscGetNumOfColumns(pTableMeta)); @@ -742,6 +758,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->queryType = htonl(pQueryInfo->type); pQueryMsg->vgroupLimit = htobe64(pQueryInfo->vgroupLimit); pQueryMsg->sqlstrLen = htonl(sqlLen); + pQueryMsg->prevResultLen = htonl(pQueryInfo->bufLen); + pQueryMsg->sw.gap = htobe64(pQueryInfo->sessionWindow.gap); + pQueryMsg->sw.primaryColId = htonl(PRIMARYKEY_TIMESTAMP_COL_INDEX); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number @@ -756,8 +775,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || !isValidDataType(pColSchema->type)) { + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + + tscError("%p tid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", - pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex, + pSql, pTableMeta->id.tid, pTableMeta->id.uid, n, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex, pColSchema->name); return TSDB_CODE_TSC_INVALID_SQL; } @@ -799,6 +822,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); + // the queried table has been removed and a new table with the same name has already been created already + // return error msg + if (pExpr->uid != pTableMeta->id.uid) { + tscError("%p table has already been destroyed", pSql); + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { tscError("%p table schema is not matched with parsed sql", pSql); return TSDB_CODE_TSC_INVALID_SQL; @@ -810,13 +840,31 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSqlFuncExpr->colInfo.colIndex = htons(pExpr->colInfo.colIndex); pSqlFuncExpr->colInfo.flag = htons(pExpr->colInfo.flag); + if (TSDB_COL_IS_UD_COL(pExpr->colInfo.flag)) { + pSqlFuncExpr->colType = htons(pExpr->resType); + pSqlFuncExpr->colBytes = htons(pExpr->resBytes); + } else if (pExpr->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + SSchema *s = tGetTbnameColumnSchema(); + + pSqlFuncExpr->colType = htons(s->type); + pSqlFuncExpr->colBytes = htons(s->bytes); + } else if (pExpr->colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SSchema s = tGetBlockDistColumnSchema(); + + pSqlFuncExpr->colType = htons(s.type); + pSqlFuncExpr->colBytes = htons(s.bytes); + } else { + SSchema* s = tscGetColumnSchemaById(pTableMeta, pExpr->colInfo.colId); + pSqlFuncExpr->colType = htons(s->type); + pSqlFuncExpr->colBytes = htons(s->bytes); + } + pSqlFuncExpr->functionId = htons(pExpr->functionId); pSqlFuncExpr->numOfParams = htons(pExpr->numOfParams); pSqlFuncExpr->resColId = htons(pExpr->resColId); pMsg += sizeof(SSqlFuncMsg); - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { - // todo add log + for (int32_t j = 0; j < pExpr->numOfParams; ++j) { // todo add log pSqlFuncExpr->arg[j].argType = htons((uint16_t)pExpr->param[j].nType); pSqlFuncExpr->arg[j].argBytes = htons(pExpr->param[j].nLen); @@ -841,39 +889,46 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < output; ++i) { SInternalField* pField = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i); SSqlExpr *pExpr = pField->pSqlExpr; + + // this should be switched to projection query if (pExpr != NULL) { + // the queried table has been removed and a new table with the same name has already been created already + // return error msg + if (pExpr->uid != pTableMeta->id.uid) { + tscError("%p table has already been destroyed", pSql); + return TSDB_CODE_TSC_INVALID_TABLE_NAME; + } + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { tscError("%p table schema is not matched with parsed sql", pSql); return TSDB_CODE_TSC_INVALID_SQL; } - pSqlFuncExpr1->colInfo.colId = htons(pExpr->colInfo.colId); - pSqlFuncExpr1->colInfo.colIndex = htons(pExpr->colInfo.colIndex); - pSqlFuncExpr1->colInfo.flag = htons(pExpr->colInfo.flag); - - pSqlFuncExpr1->functionId = htons(pExpr->functionId); - pSqlFuncExpr1->numOfParams = htons(pExpr->numOfParams); - pMsg += sizeof(SSqlFuncMsg); - - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { - // todo add log - pSqlFuncExpr1->arg[j].argType = htons((uint16_t)pExpr->param[j].nType); - pSqlFuncExpr1->arg[j].argBytes = htons(pExpr->param[j].nLen); - - if (pExpr->param[j].nType == TSDB_DATA_TYPE_BINARY) { - memcpy(pMsg, pExpr->param[j].pz, pExpr->param[j].nLen); - pMsg += pExpr->param[j].nLen; - } else { - pSqlFuncExpr1->arg[j].argValue.i64 = htobe64(pExpr->param[j].i64); + pSqlFuncExpr1->numOfParams = 0; // no params for projection query + pSqlFuncExpr1->functionId = htons(TSDB_FUNC_PRJ); + pSqlFuncExpr1->colInfo.colId = htons(pExpr->resColId); + pSqlFuncExpr1->colInfo.flag = htons(TSDB_COL_NORMAL); + + bool assign = false; + for (int32_t f = 0; f < tscSqlExprNumOfExprs(pQueryInfo); ++f) { + SSqlExpr *pe = tscSqlExprGet(pQueryInfo, f); + if (pe == pExpr) { + pSqlFuncExpr1->colInfo.colIndex = htons(f); + pSqlFuncExpr1->colType = htons(pe->resType); + pSqlFuncExpr1->colBytes = htons(pe->resBytes); + assign = true; + break; } } + assert(assign); + pMsg += sizeof(SSqlFuncMsg); pSqlFuncExpr1 = (SSqlFuncMsg *)pMsg; } else { assert(pField->pArithExprInfo != NULL); SExprInfo* pExprInfo = pField->pArithExprInfo; - pSqlFuncExpr1->colInfo.colId = htons(pExprInfo->base.colInfo.colId); + pSqlFuncExpr1->colInfo.colId = htons(pExprInfo->base.colInfo.colId); pSqlFuncExpr1->functionId = htons(pExprInfo->base.functionId); pSqlFuncExpr1->numOfParams = htons(pExprInfo->base.numOfParams); pMsg += sizeof(SSqlFuncMsg); @@ -898,8 +953,13 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->secondStageOutput = 0; } + int32_t succeed = 1; + // serialize the table info (sid, uid, tags) - pMsg = doSerializeTableInfo(pQueryMsg, pSql, pMsg); + pMsg = doSerializeTableInfo(pQueryMsg, pSql, pMsg, &succeed); + if (succeed == 0) { + return TSDB_CODE_TSC_APP_ERROR; + } SSqlGroupbyExpr *pGroupbyExpr = &pQueryInfo->groupbyExpr; if (pGroupbyExpr->numOfGroupCols > 0) { @@ -943,9 +1003,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if ((pCol->colIndex.columnIndex >= numOfTagColumns || pCol->colIndex.columnIndex < -1) || (!isValidDataType(pColSchema->type))) { + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + tscError("%p tid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s", - pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, total, numOfTagColumns, - pCol->colIndex.columnIndex, pColSchema->name); + pSql, pTableMeta->id.tid, pTableMeta->id.uid, n, total, numOfTagColumns, pCol->colIndex.columnIndex, pColSchema->name); return TSDB_CODE_TSC_INVALID_SQL; } @@ -974,6 +1036,11 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } + if (pQueryInfo->bufLen > 0) { + memcpy(pMsg, pQueryInfo->buf, pQueryInfo->bufLen); + pMsg += pQueryInfo->bufLen; + } + SCond* pCond = &pQueryInfo->tagCond.tbnameCond; if (pCond->len > 0) { strncpy(pMsg, pCond->cond, pCond->len); @@ -1016,13 +1083,15 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SCreateDbMsg); - pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DB; + + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_CREATE_DB : TSDB_MSG_TYPE_CM_CREATE_TP; SCreateDbMsg *pCreateDbMsg = (SCreateDbMsg *)pCmd->payload; assert(pCmd->numOfClause == 1); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pCreateDbMsg->db, pTableMetaInfo->name, sizeof(pCreateDbMsg->db)); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateDbMsg->db); + assert(code == TSDB_CODE_SUCCESS); return TSDB_CODE_SUCCESS; } @@ -1036,7 +1105,9 @@ int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SCreateDnodeMsg *pCreate = (SCreateDnodeMsg *)pCmd->payload; - strncpy(pCreate->ep, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); + + SStrToken* t0 = taosArrayGet(pInfo->pMiscInfo->a, 0); + strncpy(pCreate->ep, t0->z, t0->n); pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_DNODE; @@ -1053,13 +1124,13 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCreateAcctMsg *pAlterMsg = (SCreateAcctMsg *)pCmd->payload; - SStrToken *pName = &pInfo->pDCLInfo->user.user; - SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken *pName = &pInfo->pMiscInfo->user.user; + SStrToken *pPwd = &pInfo->pMiscInfo->user.passwd; strncpy(pAlterMsg->user, pName->z, pName->n); strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); - SCreateAcctSQL *pAcctOpt = &pInfo->pDCLInfo->acctOpt; + SCreateAcctInfo *pAcctOpt = &pInfo->pMiscInfo->acctOpt; pAlterMsg->cfg.maxUsers = htonl(pAcctOpt->maxUsers); pAlterMsg->cfg.maxDbs = htonl(pAcctOpt->maxDbs); @@ -1099,7 +1170,7 @@ int32_t tscBuildUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCreateUserMsg *pAlterMsg = (SCreateUserMsg *)pCmd->payload; - SUserInfo *pUser = &pInfo->pDCLInfo->user; + SUserInfo *pUser = &pInfo->pMiscInfo->user; strncpy(pAlterMsg->user, pUser->user.z, pUser->user.n); pAlterMsg->flag = (int8_t)pUser->type; @@ -1139,10 +1210,13 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SDropDbMsg *pDropDbMsg = (SDropDbMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pDropDbMsg->db, pTableMetaInfo->name, sizeof(pDropDbMsg->db)); - pDropDbMsg->ignoreNotExists = pInfo->pDCLInfo->existsCheck ? 1 : 0; - pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DB; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pDropDbMsg->db); + assert(code == TSDB_CODE_SUCCESS && pTableMetaInfo->name.type == TSDB_DB_NAME_T); + + pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + + pCmd->msgType = (pInfo->pMiscInfo->dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_DROP_DB : TSDB_MSG_TYPE_CM_DROP_TP; return TSDB_CODE_SUCCESS; } @@ -1157,15 +1231,19 @@ int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMDropTableMsg *pDropTableMsg = (SCMDropTableMsg*)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - strcpy(pDropTableMsg->tableId, pTableMetaInfo->name); - pDropTableMsg->igNotExists = pInfo->pDCLInfo->existsCheck ? 1 : 0; + tNameExtractFullName(&pTableMetaInfo->name, pDropTableMsg->name); + pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_TABLE; return TSDB_CODE_SUCCESS; } int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; + + char dnodeEp[TSDB_EP_LEN] = {0}; + tstrncpy(dnodeEp, pCmd->payload, TSDB_EP_LEN); + pCmd->payloadLen = sizeof(SDropDnodeMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); @@ -1173,60 +1251,62 @@ int32_t tscBuildDropDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SDropDnodeMsg * pDrop = (SDropDnodeMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pDrop->ep, pTableMetaInfo->name, sizeof(pDrop->ep)); + tstrncpy(pDrop->ep, dnodeEp, tListLen(pDrop->ep)); pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_DNODE; return TSDB_CODE_SUCCESS; } -int32_t tscBuildDropUserMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +int32_t tscBuildDropUserAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; + + char user[TSDB_USER_LEN] = {0}; + tstrncpy(user, pCmd->payload, TSDB_USER_LEN); + pCmd->payloadLen = sizeof(SDropUserMsg); - pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_USER; + pCmd->msgType = (pInfo->type == TSDB_SQL_DROP_USER)? TSDB_MSG_TYPE_CM_DROP_USER:TSDB_MSG_TYPE_CM_DROP_ACCT; if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user)); + SDropUserMsg *pDropMsg = (SDropUserMsg *)pCmd->payload; + tstrncpy(pDropMsg->user, user, tListLen(user)); return TSDB_CODE_SUCCESS; } -int32_t tscBuildDropAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SDropUserMsg); - pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_ACCT; + pCmd->payloadLen = sizeof(SUseDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SDropUserMsg * pDropMsg = (SDropUserMsg *)pCmd->payload; + SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pDropMsg->user, pTableMetaInfo->name, sizeof(pDropMsg->user)); + tNameExtractFullName(&pTableMetaInfo->name, pUseDbMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB; return TSDB_CODE_SUCCESS; } -int32_t tscBuildUseDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { +int32_t tscBuildSyncDbReplicaMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; - pCmd->payloadLen = sizeof(SUseDbMsg); + pCmd->payloadLen = sizeof(SSyncDbMsg); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, pCmd->payloadLen)) { tscError("%p failed to malloc for query msg", pSql); return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SUseDbMsg *pUseDbMsg = (SUseDbMsg *)pCmd->payload; + SSyncDbMsg *pSyncMsg = (SSyncDbMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - strcpy(pUseDbMsg->db, pTableMetaInfo->name); - pCmd->msgType = TSDB_MSG_TYPE_CM_USE_DB; + tNameExtractFullName(&pTableMetaInfo->name, pSyncMsg->db); + pCmd->msgType = TSDB_MSG_TYPE_CM_SYNC_DB; return TSDB_CODE_SUCCESS; } @@ -1245,14 +1325,16 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - size_t nameLen = strlen(pTableMetaInfo->name); - if (nameLen > 0) { - tstrncpy(pShowMsg->db, pTableMetaInfo->name, sizeof(pShowMsg->db)); // prefix is set here + + if (tNameIsEmpty(&pTableMetaInfo->name)) { + pthread_mutex_lock(&pObj->mutex); + tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); + pthread_mutex_unlock(&pObj->mutex); } else { - tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db)); + tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db); } - SShowInfo *pShowInfo = &pInfo->pDCLInfo->showOpt; + SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt; pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { @@ -1295,7 +1377,7 @@ int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &(pSql->cmd); int32_t size = minMsgSize() + sizeof(SCMCreateTableMsg) + sizeof(SCreateTableMsg); - SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo; + SCreateTableSql *pCreateTableInfo = pInfo->pCreateTableInfo; if (pCreateTableInfo->type == TSQL_CREATE_TABLE_FROM_STABLE) { int32_t numOfTables = (int32_t)taosArrayGetSize(pInfo->pCreateTableInfo->childTableInfo); size += numOfTables * (sizeof(SCreateTableMsg) + TSDB_MAX_TAGS_LEN); @@ -1304,19 +1386,19 @@ int tscEstimateCreateTableMsgLength(SSqlObj *pSql, SSqlInfo *pInfo) { } if (pCreateTableInfo->pSelect != NULL) { - size += (pCreateTableInfo->pSelect->selectToken.n + 1); + size += (pCreateTableInfo->pSelect->sqlstr.n + 1); } return size + TSDB_EXTRA_PAYLOAD_SIZE; } int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - int msgLen = 0; - SSchema * pSchema; - int size = 0; + int msgLen = 0; + int size = 0; + SSchema *pSchema; SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // Reallocate the payload size @@ -1347,11 +1429,10 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg += sizeof(SCreateTableMsg); SCreatedTableInfo* p = taosArrayGet(list, i); - strcpy(pCreate->tableId, p->fullname); + strcpy(pCreate->tableName, p->fullname); pCreate->igExists = (p->igExist)? 1 : 0; // use dbinfo from table id without modifying current db info - tscGetDBInfoFromTableFullName(p->fullname, pCreate->db); pMsg = serializeTagData(&p->tagdata, pMsg); int32_t len = (int32_t)(pMsg - (char*) pCreate); @@ -1360,12 +1441,10 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } else { // create (super) table pCreateTableMsg->numOfTables = htonl(1); // only one table will be created - strcpy(pCreateMsg->tableId, pTableMetaInfo->name); + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pCreateMsg->tableName); + assert(code == 0); - // use dbinfo from table id without modifying current db info - tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pCreateMsg->db); - - SCreateTableSQL *pCreateTable = pInfo->pCreateTableInfo; + SCreateTableSql *pCreateTable = pInfo->pCreateTableInfo; pCreateMsg->igExists = pCreateTable->existCheck ? 1 : 0; pCreateMsg->numOfColumns = htons(pCmd->numOfCols); @@ -1388,11 +1467,11 @@ int tscBuildCreateTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pMsg = (char *)pSchema; if (type == TSQL_CREATE_STREAM) { // check if it is a stream sql - SQuerySQL *pQuerySql = pInfo->pCreateTableInfo->pSelect; + SQuerySqlNode *pQuerySql = pInfo->pCreateTableInfo->pSelect; - strncpy(pMsg, pQuerySql->selectToken.z, pQuerySql->selectToken.n + 1); - pCreateMsg->sqlLen = htons(pQuerySql->selectToken.n + 1); - pMsg += pQuerySql->selectToken.n + 1; + strncpy(pMsg, pQuerySql->sqlstr.z, pQuerySql->sqlstr.n + 1); + pCreateMsg->sqlLen = htons(pQuerySql->sqlstr.n + 1); + pMsg += pQuerySql->sqlstr.n + 1; } } @@ -1421,7 +1500,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - SAlterTableSQL *pAlterInfo = pInfo->pAlterInfo; + SAlterTableInfo *pAlterInfo = pInfo->pAlterInfo; int size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for alter table msg", pSql); @@ -1429,9 +1508,8 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SAlterTableMsg *pAlterTableMsg = (SAlterTableMsg *)pCmd->payload; - tscGetDBInfoFromTableFullName(pTableMetaInfo->name, pAlterTableMsg->db); - strcpy(pAlterTableMsg->tableId, pTableMetaInfo->name); + tNameExtractFullName(&pTableMetaInfo->name, pAlterTableMsg->tableFname); pAlterTableMsg->type = htons(pAlterInfo->type); pAlterTableMsg->numOfCols = htons(tscNumOfFields(pQueryInfo)); @@ -1482,11 +1560,13 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { int tscAlterDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; pCmd->payloadLen = sizeof(SAlterDbMsg); - pCmd->msgType = TSDB_MSG_TYPE_CM_ALTER_DB; + pCmd->msgType = (pInfo->pMiscInfo->dbOpt.dbType == TSDB_DB_TYPE_DEFAULT) ? TSDB_MSG_TYPE_CM_ALTER_DB : TSDB_MSG_TYPE_CM_ALTER_TP; SAlterDbMsg *pAlterDbMsg = (SAlterDbMsg* )pCmd->payload; + pAlterDbMsg->dbType = -1; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tstrncpy(pAlterDbMsg->db, pTableMetaInfo->name, sizeof(pAlterDbMsg->db)); + tNameExtractFullName(&pTableMetaInfo->name, pAlterDbMsg->db); return TSDB_CODE_SUCCESS; } @@ -1503,7 +1583,7 @@ int tscBuildRetrieveFromMgmtMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); SRetrieveTableMsg *pRetrieveMsg = (SRetrieveTableMsg*)pCmd->payload; - pRetrieveMsg->qhandle = htobe64(pSql->res.qhandle); + pRetrieveMsg->qId = htobe64(pSql->res.qId); pRetrieveMsg->free = htons(pQueryInfo->type); return TSDB_CODE_SUCCESS; @@ -1611,9 +1691,14 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // TODO refactor full_name char *db; // ugly code to move the space + + pthread_mutex_lock(&pObj->mutex); db = strstr(pObj->db, TS_PATH_DELIMITER); + db = (db == NULL) ? pObj->db : db + 1; tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + tstrncpy(pConnect->clientVersion, version, sizeof(pConnect->clientVersion)); tstrncpy(pConnect->msgVersion, "", sizeof(pConnect->msgVersion)); @@ -1624,13 +1709,17 @@ int tscBuildConnectMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { - SSqlCmd * pCmd = &pSql->cmd; + SSqlCmd *pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload; + + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pInfoMsg->tableFname); + if (code != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } - STableInfoMsg *pInfoMsg = (STableInfoMsg *)pCmd->payload; - strcpy(pInfoMsg->tableId, pTableMetaInfo->name); pInfoMsg->createFlag = htons(pSql->cmd.autoCreated ? 1 : 0); char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); @@ -1687,33 +1776,6 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { return 0; } -//static UNUSED_FUNC int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { -//// const int32_t defaultSize = -//// minMsgSize() + sizeof(SSuperTableMetaMsg) + sizeof(SMgmtHead) + sizeof(int16_t) * TSDB_MAX_TAGS; -//// SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); -//// -//// int32_t n = 0; -//// size_t size = taosArrayGetSize(pQueryInfo->tagCond.pCond); -//// for (int32_t i = 0; i < size; ++i) { -//// assert(0); -////// n += strlen(pQueryInfo->tagCond.cond[i].cond); -//// } -//// -//// int32_t tagLen = n * TSDB_NCHAR_SIZE; -//// if (pQueryInfo->tagCond.tbnameCond.cond != NULL) { -//// tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE; -//// } -//// -//// int32_t joinCondLen = (TSDB_TABLE_FNAME_LEN + sizeof(int16_t)) * 2; -//// int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; -//// -//// int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); -//// -//// int32_t len = tagLen + joinCondLen + elemSize + colSize + defaultSize; -//// -//// return MAX(len, TSDB_DEFAULT_PAYLOAD_SIZE); -//} - int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SSqlCmd *pCmd = &pSql->cmd; @@ -1726,9 +1788,10 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); - size_t size = sizeof(pTableMetaInfo->name); - tstrncpy(pMsg, pTableMetaInfo->name, size); - pMsg += size; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, pMsg); + assert(code == TSDB_CODE_SUCCESS); + + pMsg += TSDB_TABLE_FNAME_LEN; } pCmd->msgType = TSDB_MSG_TYPE_CM_STABLE_VGROUP; @@ -1795,11 +1858,11 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->uid = htobe64(pMetaMsg->uid); pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - + if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, - pMetaMsg->tid, pMetaMsg->tableId); + pMetaMsg->tid, pMetaMsg->tableFname); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1828,15 +1891,18 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { assert(i == 0); } - assert(isValidDataType(pSchema->type)); pSchema++; } - - STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); assert(pTableMetaInfo->pTableMeta == NULL); + STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg); + if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) { + tscError("%p invalid table meta from mnode, name:%s", pSql, tNameGetTableName(&pTableMetaInfo->name)); + return TSDB_CODE_TSC_INVALID_VALUE; + } + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { // check if super table hashmap or not int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); @@ -1851,11 +1917,19 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { tfree(pSupTableMeta); CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta); - taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), cMeta, sizeof(CChildTableMeta)); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + taosHashPut(tscTableMetaInfo, name, strlen(name), cMeta, sizeof(CChildTableMeta)); tfree(cMeta); } else { uint32_t s = tscGetTableMetaSize(pTableMeta); - taosHashPut(tscTableMetaInfo, pTableMetaInfo->name, strlen(pTableMetaInfo->name), pTableMeta, s); + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + taosHashPut(tscTableMetaInfo, name, strlen(name), pTableMeta, s); } // update the vgroupInfo if needed @@ -1874,9 +1948,10 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } } - tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name); + tscDebug("%p recv table meta, uid:%" PRIu64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, + tNameGetTableName(&pTableMetaInfo->name)); + free(pTableMeta); - return TSDB_CODE_SUCCESS; } @@ -2016,19 +2091,24 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { assert(pInfo->vgroupList != NULL); pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups; - for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { - //just init, no need to lock - SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; + if (pInfo->vgroupList->numOfVgroups <= 0) { + //tfree(pInfo->vgroupList); + tscError("%p empty vgroup info", pSql); + } else { + for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { + //just init, no need to lock + SVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; - pVgroups->vgId = htonl(vmsg->vgId); - pVgroups->numOfEps = vmsg->numOfEps; + SVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + pVgroups->vgId = htonl(vmsg->vgId); + pVgroups->numOfEps = vmsg->numOfEps; - assert(pVgroups->numOfEps >= 1 && pVgroups->vgId >= 1); + assert(pVgroups->numOfEps >= 1 && pVgroups->vgId >= 1); - for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { - pVgroups->epAddr[k].port = htons(vmsg->epAddr[k].port); - pVgroups->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, tListLen(vmsg->epAddr[k].fqdn)); + for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { + pVgroups->epAddr[k].port = htons(vmsg->epAddr[k].port); + pVgroups->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, tListLen(vmsg->epAddr[k].fqdn)); + } } } @@ -2054,7 +2134,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { pShow = (SShowRsp *)pRes->pRsp; pShow->qhandle = htobe64(pShow->qhandle); - pRes->qhandle = pShow->qhandle; + pRes->qId = pShow->qhandle; tscResetForNextRetrieve(pRes); pMetaMsg = &(pShow->tableMeta); @@ -2097,7 +2177,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { return 0; } -static void createHBObj(STscObj* pObj) { +static void createHbObj(STscObj* pObj) { if (pObj->hbrid != 0) { return; } @@ -2141,10 +2221,13 @@ int tscProcessConnectRsp(SSqlObj *pSql) { SConnectRsp *pConnect = (SConnectRsp *)pRes->pRsp; tstrncpy(pObj->acctId, pConnect->acctId, sizeof(pObj->acctId)); // copy acctId from response + + pthread_mutex_lock(&pObj->mutex); int32_t len = sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); + pthread_mutex_unlock(&pObj->mutex); if (pConnect->epSet.numOfEps > 0) { tscEpSetHtons(&pConnect->epSet); @@ -2160,7 +2243,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { pObj->superAuth = pConnect->superAuth; pObj->connId = htonl(pConnect->connId); - createHBObj(pObj); + createHbObj(pObj); //launch a timer to send heartbeat to maintain the connection and send status to mnode taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer); @@ -2171,13 +2254,18 @@ int tscProcessConnectRsp(SSqlObj *pSql) { int tscProcessUseDbRsp(SSqlObj *pSql) { STscObj * pObj = pSql->pTscObj; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - - tstrncpy(pObj->db, pTableMetaInfo->name, sizeof(pObj->db)); - return 0; + + pthread_mutex_lock(&pObj->mutex); + int ret = tNameExtractFullName(&pTableMetaInfo->name, pObj->db); + pthread_mutex_unlock(&pObj->mutex); + + return ret; } int tscProcessDropDbRsp(SSqlObj *pSql) { - pSql->pTscObj->db[0] = 0; + //TODO LOCK DB WHEN MODIFY IT + //pSql->pTscObj->db[0] = 0; + taosHashEmpty(tscTableMetaInfo); return 0; } @@ -2186,18 +2274,22 @@ int tscProcessDropTableRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); //The cached tableMeta is expired in this case, so clean it in hash table - taosHashRemove(tscTableMetaInfo, pTableMetaInfo->name, strnlen(pTableMetaInfo->name, TSDB_TABLE_FNAME_LEN)); - tscDebug("%p remove table meta after drop table:%s, numOfRemain:%d", pSql, pTableMetaInfo->name, - (int32_t) taosHashGetSize(tscTableMetaInfo)); + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); - assert(pTableMetaInfo->pTableMeta == NULL); + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + tscDebug("%p remove table meta after drop table:%s, numOfRemain:%d", pSql, name, (int32_t) taosHashGetSize(tscTableMetaInfo)); + + pTableMetaInfo->pTableMeta = NULL; return 0; } int tscProcessAlterTableMsgRsp(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - char* name = pTableMetaInfo->name; + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + tscDebug("%p remove tableMeta in hashMap after alter-table: %s", pSql, name); bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); @@ -2224,11 +2316,12 @@ int tscProcessQueryRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SQueryTableRsp *pQuery = (SQueryTableRsp *)pRes->pRsp; - pQuery->qhandle = htobe64(pQuery->qhandle); - pRes->qhandle = pQuery->qhandle; + pQuery->qId = htobe64(pQuery->qId); + pRes->qId = pQuery->qId; pRes->data = NULL; tscResetForNextRetrieve(pRes); + tscDebug("%p query rsp received, qId:%"PRIu64, pSql, pRes->qId); return 0; } @@ -2236,6 +2329,8 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + assert(pRes->rspLen >= sizeof(SRetrieveTableRsp)); + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; if (pRetrieve == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -2284,7 +2379,8 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { } pRes->row = 0; - tscDebug("%p numOfRows:%d, offset:%" PRId64 ", complete:%d", pSql, pRes->numOfRows, pRes->offset, pRes->completed); + tscDebug("%p numOfRows:%d, offset:%" PRId64 ", complete:%d, qId:%"PRIu64, pSql, pRes->numOfRows, pRes->offset, + pRes->completed, pRes->qId); return 0; } @@ -2316,7 +2412,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn STableMetaInfo *pNewMeterMetaInfo = tscAddEmptyMetaInfo(pNewQueryInfo); assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); - tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name)); + tNameAssign(&pNewMeterMetaInfo->name, &pTableMetaInfo->name); if (pSql->cmd.autoCreated) { int32_t code = copyTagData(&pNew->cmd.tagData, &pSql->cmd.tagData); @@ -2347,7 +2443,8 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn } int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { - assert(strlen(pTableMetaInfo->name) != 0); + assert(tIsValidName(&pTableMetaInfo->name)); + tfree(pTableMetaInfo->pTableMeta); uint32_t size = tscGetTableMetaMaxSize(); @@ -2355,15 +2452,18 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { pTableMetaInfo->pTableMeta->tableType = -1; pTableMetaInfo->pTableMeta->tableInfo.numOfColumns = -1; - int32_t len = (int32_t) strlen(pTableMetaInfo->name); - taosHashGetClone(tscTableMetaInfo, pTableMetaInfo->name, len, NULL, pTableMetaInfo->pTableMeta, -1); + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + size_t len = strlen(name); + taosHashGetClone(tscTableMetaInfo, name, len, NULL, pTableMetaInfo->pTableMeta, -1); // TODO resize the tableMeta STableMeta* pMeta = pTableMetaInfo->pTableMeta; if (pMeta->id.uid > 0) { if (pMeta->tableType == TSDB_CHILD_TABLE) { - int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, pTableMetaInfo->name); + int32_t code = tscCreateTableMetaFromCChildMeta(pTableMetaInfo->pTableMeta, name); if (code != TSDB_CODE_SUCCESS) { return getTableMetaFromMnode(pSql, pTableMetaInfo); } @@ -2391,16 +2491,24 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - const char* name = pTableMetaInfo->name; + + char name[TSDB_TABLE_FNAME_LEN] = {0}; + int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name); + if (code != TSDB_CODE_SUCCESS) { + tscError("%p failed to generate the table full name", pSql); + return TSDB_CODE_TSC_INVALID_SQL; + } STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMeta) { tscDebug("%p update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64, pSql, name, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid); } // remove stored tableMeta info in hash table - taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + size_t len = strlen(name); + taosHashRemove(tscTableMetaInfo, name, len); + return getTableMetaFromMnode(pSql, pTableMetaInfo); } @@ -2441,8 +2549,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); - STableMeta* pTableMeta = tscTableMetaClone(pMInfo->pTableMeta); - tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); + STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta); + tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -2482,9 +2590,10 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg; tscBuildMsg[TSDB_SQL_CREATE_TABLE] = tscBuildCreateTableMsg; - tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserMsg; - tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg; + tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg; tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg; + tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg; tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg; tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg; tscBuildMsg[TSDB_SQL_CREATE_DNODE] = tscBuildCreateDnodeMsg; @@ -2538,7 +2647,6 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; - tscKeepConn[TSDB_SQL_SHOW] = 1; tscKeepConn[TSDB_SQL_RETRIEVE] = 1; tscKeepConn[TSDB_SQL_SELECT] = 1; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 3f25d7a14d90a4d8aac6ca33b53d34f721b9847b..93d0e9fd092b7e91fc8028e30f73a13bdb02627d 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -15,7 +15,7 @@ #include "hash.h" #include "os.h" -#include "qAst.h" +#include "texpr.h" #include "tkey.h" #include "tcache.h" #include "tnote.h" @@ -52,7 +52,9 @@ static bool validPassword(const char* passwd) { static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { - taos_init(); + if (taos_init()) { + return NULL; + } if (!validUserName(user)) { terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; @@ -90,9 +92,11 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa } else { if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond, &corMgmtEpSet) < 0) return NULL; } + char rpcKey[512] = {0}; + snprintf(rpcKey, sizeof(rpcKey), "%s:%s:%s:%d", user, pass, ip, port); - void *pDnodeConn = NULL; - if (tscInitRpc(user, secretEncrypt, &pDnodeConn) != 0) { + void *pRpcObj = NULL; + if (tscAcquireRpc(rpcKey, user, secretEncrypt, &pRpcObj) != 0) { terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return NULL; } @@ -100,22 +104,21 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - rpcClose(pDnodeConn); + tscReleaseRpc(pRpcObj); return NULL; } - // set up tscObj's mgmtEpSet - pObj->tscCorMgmtEpSet = (SRpcCorEpSet *)malloc(sizeof(SRpcCorEpSet)); - if (NULL == pObj->tscCorMgmtEpSet) { + + pObj->tscCorMgmtEpSet = malloc(sizeof(SRpcCorEpSet)); + if (pObj->tscCorMgmtEpSet == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - rpcClose(pDnodeConn); - free(pObj->tscCorMgmtEpSet); + tscReleaseRpc(pRpcObj); free(pObj); + return NULL; } - memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(SRpcCorEpSet)); - - pObj->signature = pObj; - pObj->pDnodeConn = pDnodeConn; + memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(corMgmtEpSet)); + pObj->signature = pObj; + pObj->pRpcObj = (SRpcObj *)pRpcObj; tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); memcpy(pObj->pass, secretEncrypt, secretEncryptLen); @@ -125,8 +128,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa /* db name is too long */ if (len >= TSDB_DB_NAME_LEN) { terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; - rpcClose(pDnodeConn); - free(pObj->tscCorMgmtEpSet); + tscReleaseRpc(pRpcObj); free(pObj); return NULL; } @@ -143,8 +145,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj)); if (NULL == pSql) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - rpcClose(pDnodeConn); - free(pObj->tscCorMgmtEpSet); + tscReleaseRpc(pRpcObj); free(pObj); return NULL; } @@ -160,9 +161,8 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - rpcClose(pDnodeConn); + tscReleaseRpc(pRpcObj); free(pSql); - free(pObj->tscCorMgmtEpSet); free(pObj); return NULL; } @@ -201,7 +201,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, return NULL; } - tscDebug("%p DB connection is opening, dnodeConn:%p", pObj, pObj->pDnodeConn); + tscDebug("%p DB connection is opening, rpcObj: %p, dnodeConn:%p", pObj, pObj->pRpcObj, pObj->pRpcObj->pDnodeConn); taos_free_result(pSql); // version compare only requires the first 3 segments of the version string @@ -278,7 +278,7 @@ void taos_close(TAOS *taos) { return; } - tscDebug("%p try to free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); + tscDebug("%p try to free tscObj", pObj); if (pObj->signature != pObj) { tscDebug("%p already closed or invalid tscObj", pObj); return; @@ -294,11 +294,15 @@ void taos_close(TAOS *taos) { tscDebug("%p HB is freed", pHb); taosReleaseRef(tscObjRef, pHb->self); +#ifdef __APPLE__ + // to satisfy later tsem_destroy in taos_free_result + tsem_init(&pHb->rspSem, 0, 0); +#endif // __APPLE__ taos_free_result(pHb); } } - tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); + tscDebug("%p all sqlObj are freed, free tscObj", pObj); taosRemoveRef(tscRefId, pObj->rid); } @@ -441,24 +445,6 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pFieldInfo->final; } -int taos_retrieve(TAOS_RES *res) { - if (res == NULL) return 0; - SSqlObj *pSql = (SSqlObj *)res; - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; - if (pSql == NULL || pSql->signature != pSql) return 0; - if (pRes->qhandle == 0) return 0; - - tscResetForNextRetrieve(pRes); - - if (pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; - } - - tscProcessSql(pSql); - return pRes->numOfRows; -} - static bool needToFetchNewBlock(SSqlObj* pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -490,7 +476,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0 || + if (pRes->qId == 0 || pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { @@ -522,7 +508,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0 || + if (pRes->qId == 0 || pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { @@ -568,7 +554,7 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - if (pRes == NULL || pRes->qhandle == 0) { + if (pRes == NULL || pRes->qId == 0) { return true; } @@ -712,7 +698,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) { } tscAsyncResultOnError(pSubObj); - taosReleaseRef(tscObjRef, pSubObj->self); + // taosRelekaseRef(tscObjRef, pSubObj->self); } if (pSql->subState.numOfSub <= 0) { @@ -781,6 +767,7 @@ bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { int len = 0; + for (int i = 0; i < num_fields; ++i) { if (i > 0) { str[len++] = ' '; @@ -838,13 +825,15 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { - size_t xlen = 0; - for (xlen = 0; xlen < fields[i].bytes - VARSTR_HEADER_SIZE; xlen++) { - char c = ((char *)row[i])[xlen]; - if (c == 0) break; - str[len++] = c; + int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE); + if (fields[i].type == TSDB_DATA_TYPE_BINARY) { + assert(charLen <= fields[i].bytes); + } else { + assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE); } - str[len] = 0; + + memcpy(str + len, row[i], charLen); + len += charLen; } break; case TSDB_DATA_TYPE_TIMESTAMP: @@ -933,7 +922,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) { // must before clean the sqlcmd object - tscResetSqlCmdObj(&pSql->cmd); + tscResetSqlCmd(&pSql->cmd, false); SSqlCmd *pCmd = &pSql->cmd; @@ -992,7 +981,8 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t return code; } - if (payloadLen + strlen(pTableMetaInfo->name) + 128 >= pCmd->allocSize) { + int32_t xlen = tNameLen(&pTableMetaInfo->name); + if (payloadLen + xlen + 128 >= pCmd->allocSize) { char *pNewMem = realloc(pCmd->payload, pCmd->allocSize + tblListLen); if (pNewMem == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1005,7 +995,9 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t pMsg = pCmd->payload; } - payloadLen += sprintf(pMsg + payloadLen, "%s,", pTableMetaInfo->name); + char n[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, n); + payloadLen += sprintf(pMsg + payloadLen, "%s,", n); } *(pMsg + payloadLen) = '\0'; @@ -1058,7 +1050,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { * If qhandle is NOT set 0, the function of taos_free_result() will send message to server by calling tscProcessSql() * to free connection, which may cause segment fault, when the parse phrase is not even successfully executed. */ - pRes->qhandle = 0; + pRes->qId = 0; free(str); if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index a39fbc94202d3c1940b8631c9f95a38381857946..7699e6f45973c8ca834e0e93ad5f5f31797ec520 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -103,8 +103,8 @@ static void doLaunchQuery(void* param, TAOS_RES* tres, int32_t code) { // failed to get table Meta or vgroup list, retry in 10sec. if (code == TSDB_CODE_SUCCESS) { - tscTansformSQLFuncForSTableQuery(pQueryInfo); - tscDebug("%p stream:%p, start stream query on:%s", pSql, pStream, pTableMetaInfo->name); + tscTansformFuncForSTableQuery(pQueryInfo); + tscDebug("%p stream:%p, start stream query on:%s", pSql, pStream, tNameGetTableName(&pTableMetaInfo->name)); pSql->fp = tscProcessStreamQueryCallback; pSql->fetchFp = tscProcessStreamQueryCallback; @@ -191,7 +191,9 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); - char* name = pTableMetaInfo->name; + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); @@ -291,12 +293,13 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf pStream->stime += 1; } - tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name, + tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, tNameGetTableName(&pTableMetaInfo->name), pStream->numOfRes); tfree(pTableMetaInfo->pTableMeta); tscFreeSqlResult(pSql); + tscFreeSubobj(pSql); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); @@ -556,7 +559,7 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) { taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer); tscDebug("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql, - pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); + pStream, tNameGetTableName(&pTableMetaInfo->name), pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr); } void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) { diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 7f0b174ad364c783d03b3e7c7ee9bf8854c27cc7..1277a436a14c926b756ff99b0cd2e045f5dfed1f 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) { SSub* pSub = (SSub*)sub; SSubscriptionProgress target = {.uid = uid, .key = 0}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); + SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); if (p == NULL) { return dflt; } @@ -76,7 +76,7 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { SSub* pSub = (SSub*)sub; SSubscriptionProgress target = {.uid = uid, .key = ts}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); + SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); if (p != NULL) { p->key = ts; tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key); @@ -149,7 +149,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* } strtolower(pSql->sqlstr, pSql->sqlstr); - pRes->qhandle = 0; + pRes->qId = 0; pRes->numOfRows = 1; code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); @@ -270,7 +270,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0}; - SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); + SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ); if (p == NULL) { taosArrayClear(pSub->progress); taosArrayPush(pSub->progress, &target); @@ -313,7 +313,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) { char buf[TSDB_MAX_SQL_LEN]; sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(buf, "r"); + FILE* fp = fopen(buf, "rb"); if (fp == NULL) { tscDebug("subscription progress file does not exist: %s", pSub->topic); return 1; @@ -368,7 +368,7 @@ void tscSaveSubscriptionProgress(void* sub) { } sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); - FILE* fp = fopen(path, "w+"); + FILE* fp = fopen(path, "wb+"); if (fp == NULL) { tscError("failed to create progress file for subscription: %s", pSub->topic); return; @@ -448,7 +448,7 @@ SSqlObj* recreateSqlObj(SSub* pSub) { return NULL; } - pRes->qhandle = 0; + pRes->qId = 0; pRes->numOfRows = 1; int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); @@ -503,9 +503,19 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { SSqlCmd *pCmd = &pSql->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single tabel subscription - pQueryInfo->window.skey = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, 0))->key; - tscDebug("subscribe:%s set subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); + if (taosArrayGetSize(pSub->progress) > 0) { // fix crash in single table subscription + + size_t size = taosArrayGetSize(pSub->progress); + TSKEY s = INT64_MAX; + for(int32_t i = 0; i < size; ++i) { + TSKEY k = ((SSubscriptionProgress*)taosArrayGet(pSub->progress, i))->key; + if (s > k) { + s = k; + } + } + + pQueryInfo->window.skey = s; + tscDebug("subscribe:%s set next round subscribe skey:%"PRId64, pSub->topic, pQueryInfo->window.skey); } if (pSub->pTimer == NULL) { @@ -536,7 +546,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { uint32_t type = pQueryInfo->type; tscFreeSqlResult(pSql); pRes->numOfRows = 1; - pRes->qhandle = 0; + pRes->qId = 0; pSql->cmd.command = TSDB_SQL_SELECT; pQueryInfo->type = type; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 2622246111894caf39f3c1c37923d1a7207935b6..299cf038057bf5ce80c449be41faa484e0a8ef93 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -13,16 +13,17 @@ * along with this program. If not, see . */ #define _GNU_SOURCE - + #include "os.h" -#include "qAst.h" +#include "texpr.h" #include "qTsbuf.h" #include "tcompare.h" #include "tscLog.h" #include "tscSubquery.h" #include "tschemautil.h" #include "tsclient.h" +#include "qUtil.h" typedef struct SInsertSupporter { SSqlObj* pSql; @@ -45,6 +46,13 @@ static int32_t tsCompare(int32_t order, int64_t left, int64_t right) { } static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { + STSElem el1 = tsBufGetElem(pTSBuf); + + int32_t res = tVariantCompare(el1.tag, tag1); + if (res != 0) { // it is a record with new tag + return; + } + while (tsBufNextPos(pTSBuf)) { STSElem el1 = tsBufGetElem(pTSBuf); @@ -55,123 +63,295 @@ static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { } } -static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJoinSupporter* pSupporter2, STimeWindow * win) { - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); +static void subquerySetState(SSqlObj *pSql, SSubqueryState *subState, int idx, int8_t state) { + assert(idx < subState->numOfSub); + assert(subState->states); + + pthread_mutex_lock(&subState->mutex); + + tscDebug("subquery:%p,%d state set to %d", pSql, idx, state); + + subState->states[idx] = state; + + pthread_mutex_unlock(&subState->mutex); +} + +static bool allSubqueryDone(SSqlObj *pParentSql) { + bool done = true; + SSubqueryState *subState = &pParentSql->subState; + + //lock in caller + tscDebug("%p total subqueries: %d", pParentSql, subState->numOfSub); + for (int i = 0; i < subState->numOfSub; i++) { + if (0 == subState->states[i]) { + tscDebug("%p subquery:%p, index: %d NOT finished, abort query completion check", pParentSql, pParentSql->pSubs[i], i); + done = false; + break; + } else { + tscDebug("%p subquery:%p, index: %d finished", pParentSql, pParentSql->pSubs[i], i); + } + } + + return done; +} + +static bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx) { + SSubqueryState *subState = &pParentSql->subState; + + assert(idx < subState->numOfSub); + + pthread_mutex_lock(&subState->mutex); + + bool done = allSubqueryDone(pParentSql); + + if (done) { + tscDebug("%p subquery:%p,%d all subs already done", pParentSql, pSql, idx); + + pthread_mutex_unlock(&subState->mutex); + + return false; + } + + tscDebug("%p subquery:%p,%d state set to 1", pParentSql, pSql, idx); + + subState->states[idx] = 1; - STSBuf* output1 = tsBufCreate(true, pQueryInfo->order.order); - STSBuf* output2 = tsBufCreate(true, pQueryInfo->order.order); + done = allSubqueryDone(pParentSql); + + pthread_mutex_unlock(&subState->mutex); + + return done; +} + + + +static int64_t doTSBlockIntersect(SSqlObj* pSql, STimeWindow * win) { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); win->skey = INT64_MAX; win->ekey = INT64_MIN; SLimitVal* pLimit = &pQueryInfo->limit; int32_t order = pQueryInfo->order.order; + int32_t joinNum = pSql->subState.numOfSub; + SMergeTsCtx ctxlist[TSDB_MAX_JOIN_TABLE_NUM] = {{0}}; + SMergeTsCtx* ctxStack[TSDB_MAX_JOIN_TABLE_NUM] = {0}; + int32_t slot = 0; + size_t tableNum = 0; + int16_t* tableMIdx = 0; + int32_t equalNum = 0; + int32_t stackidx = 0; + SMergeTsCtx* ctx = NULL; + SMergeTsCtx* pctx = NULL; + SMergeTsCtx* mainCtx = NULL; + STSElem cur; + STSElem prev; + SArray* tsCond = NULL; + int32_t mergeDone = 0; + + for (int32_t i = 0; i < joinNum; ++i) { + STSBuf* output = tsBufCreate(true, pQueryInfo->order.order); + SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(&pSql->pSubs[i]->cmd, 0); + + pSubQueryInfo->tsBuf = output; + + SJoinSupporter* pSupporter = pSql->pSubs[i]->param; - SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0); - SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0); + if (pSupporter->pTSBuf == NULL) { + tscDebug("%p at least one ts-comp is empty, 0 for secondary query after ts blocks intersecting", pSql); + return 0; + } - pSubQueryInfo1->tsBuf = output1; - pSubQueryInfo2->tsBuf = output2; + tsBufResetPos(pSupporter->pTSBuf); - TSKEY st = taosGetTimestampUs(); + if (!tsBufNextPos(pSupporter->pTSBuf)) { + tscDebug("%p input1 is empty, 0 for secondary query after ts blocks intersecting", pSql); + return 0; + } - // no result generated, return directly - if (pSupporter1->pTSBuf == NULL || pSupporter2->pTSBuf == NULL) { - tscDebug("%p at least one ts-comp is empty, 0 for secondary query after ts blocks intersecting", pSql); - return 0; + tscDebug("%p sub:%p table idx:%d, input group number:%d", pSql, pSql->pSubs[i], i, pSupporter->pTSBuf->numOfGroups); + + ctxlist[i].p = pSupporter; + ctxlist[i].res = output; } - tsBufResetPos(pSupporter1->pTSBuf); - tsBufResetPos(pSupporter2->pTSBuf); + TSKEY st = taosGetTimestampUs(); - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - tsBufFlush(output1); - tsBufFlush(output2); + for (int16_t tidx = 0; tidx < joinNum; tidx++) { + pctx = &ctxlist[tidx]; + if (pctx->compared) { + continue; + } - tscDebug("%p input1 is empty, 0 for secondary query after ts blocks intersecting", pSql); - return 0; - } + assert(pctx->numOfInput == 0); - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - tsBufFlush(output1); - tsBufFlush(output2); + tsCond = pQueryInfo->tagCond.joinInfo.joinTables[tidx]->tsJoin; - tscDebug("%p input2 is empty, 0 for secondary query after ts blocks intersecting", pSql); - return 0; - } + tableNum = taosArrayGetSize(tsCond); + assert(tableNum >= 2); + + for (int32_t i = 0; i < tableNum; ++i) { + tableMIdx = taosArrayGet(tsCond, i); + SMergeTsCtx* tctx = &ctxlist[*tableMIdx]; + tctx->compared = 1; + } - int64_t numOfInput1 = 1; - int64_t numOfInput2 = 1; + tableMIdx = taosArrayGet(tsCond, 0); + pctx = &ctxlist[*tableMIdx]; - while(1) { - STSElem elem = tsBufGetElem(pSupporter1->pTSBuf); + mainCtx = pctx; - // no data in pSupporter1 anymore, jump out of loop - if (!tsBufIsValidElem(&elem)) { - break; - } + while (1) { + pctx = mainCtx; - // find the data in supporter2 with the same tag value - STSElem e2 = tsBufFindElemStartPosByTag(pSupporter2->pTSBuf, elem.tag); + prev = tsBufGetElem(pctx->p->pTSBuf); - /** - * there are elements in pSupporter2 with the same tag, continue - */ - tVariant tag1 = {0}; - tVariantAssign(&tag1, elem.tag); + ctxStack[stackidx++] = pctx; + + if (!tsBufIsValidElem(&prev)) { + break; + } + + tVariant tag = {0}; + tVariantAssign(&tag, prev.tag); + + int32_t skipped = 0; + + for (int32_t i = 1; i < tableNum; ++i) { + SMergeTsCtx* tctx = &ctxlist[i]; + + // find the data in supporter2 with the same tag value + STSElem e2 = tsBufFindElemStartPosByTag(tctx->p->pTSBuf, &tag); + + if (!tsBufIsValidElem(&e2)) { + skipRemainValue(pctx->p->pTSBuf, &tag); + skipped = 1; + break; + } + } + + if (skipped) { + slot = 0; + stackidx = 0; + continue; + } + + tableMIdx = taosArrayGet(tsCond, ++slot); + equalNum = 1; - if (tsBufIsValidElem(&e2)) { while (1) { - STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); - STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + ctx = &ctxlist[*tableMIdx]; + + prev = tsBufGetElem(pctx->p->pTSBuf); + cur = tsBufGetElem(ctx->p->pTSBuf); // data with current are exhausted - if (!tsBufIsValidElem(&elem1) || tVariantCompare(elem1.tag, &tag1) != 0) { + if (!tsBufIsValidElem(&prev) || tVariantCompare(prev.tag, &tag) != 0) { break; } - if (!tsBufIsValidElem(&elem2) || tVariantCompare(elem2.tag, &tag1) != 0) { // ignore all records with the same tag - skipRemainValue(pSupporter1->pTSBuf, &tag1); + if (!tsBufIsValidElem(&cur) || tVariantCompare(cur.tag, &tag) != 0) { // ignore all records with the same tag break; } - /* - * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the - * final results which is acquired after the secondary merge of in the client. - */ - int32_t re = tsCompare(order, elem1.ts, elem2.ts); - if (re < 0) { - tsBufNextPos(pSupporter1->pTSBuf); - numOfInput1++; - } else if (re > 0) { - tsBufNextPos(pSupporter2->pTSBuf); - numOfInput2++; - } else { - if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - if (win->skey > elem1.ts) { - win->skey = elem1.ts; + ctxStack[stackidx++] = ctx; + + int32_t ret = tsCompare(order, prev.ts, cur.ts); + if (ret == 0) { + if (++equalNum < tableNum) { + pctx = ctx; + + if (++slot >= tableNum) { + slot = 0; } - if (win->ekey < elem1.ts) { - win->ekey = elem1.ts; + tableMIdx = taosArrayGet(tsCond, slot); + continue; + } + + assert(stackidx == tableNum); + + if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { + if (win->skey > prev.ts) { + win->skey = prev.ts; + } + + if (win->ekey < prev.ts) { + win->ekey = prev.ts; } - tsBufAppend(output1, elem1.id, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.id, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + for (int32_t i = 0; i < stackidx; ++i) { + SMergeTsCtx* tctx = ctxStack[i]; + prev = tsBufGetElem(tctx->p->pTSBuf); + + tsBufAppend(tctx->res, prev.id, prev.tag, (const char*)&prev.ts, sizeof(prev.ts)); + } } else { pLimit->offset -= 1;//offset apply to projection? } - tsBufNextPos(pSupporter1->pTSBuf); - numOfInput1++; + for (int32_t i = 0; i < stackidx; ++i) { + SMergeTsCtx* tctx = ctxStack[i]; + + if (!tsBufNextPos(tctx->p->pTSBuf) && tctx == mainCtx) { + mergeDone = 1; + } + tctx->numOfInput++; + } + + if (mergeDone) { + break; + } + + stackidx = 0; + equalNum = 1; + + ctxStack[stackidx++] = pctx; + } else if (ret > 0) { + if (!tsBufNextPos(ctx->p->pTSBuf) && ctx == mainCtx) { + mergeDone = 1; + break; + } + + ctx->numOfInput++; + stackidx--; + } else { + stackidx--; + + for (int32_t i = 0; i < stackidx; ++i) { + SMergeTsCtx* tctx = ctxStack[i]; + + if (!tsBufNextPos(tctx->p->pTSBuf) && tctx == mainCtx) { + mergeDone = 1; + } + tctx->numOfInput++; + } - tsBufNextPos(pSupporter2->pTSBuf); - numOfInput2++; + if (mergeDone) { + break; + } + + stackidx = 0; + equalNum = 1; + + ctxStack[stackidx++] = pctx; } + + } + + if (mergeDone) { + break; } - } else { // no data in pSupporter2, ignore current data in pSupporter2 - skipRemainValue(pSupporter1->pTSBuf, &tag1); + + slot = 0; + stackidx = 0; + + skipRemainValue(mainCtx->p->pTSBuf, &tag); } + + stackidx = 0; + slot = 0; + mergeDone = 0; } /* @@ -179,28 +359,32 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ * 1. only one element * 2. only one element for each tag. */ - if (output1->tsOrder == -1) { - output1->tsOrder = TSDB_ORDER_ASC; - output2->tsOrder = TSDB_ORDER_ASC; + if (ctxlist[0].res->tsOrder == -1) { + for (int32_t i = 0; i < joinNum; ++i) { + ctxlist[i].res->tsOrder = TSDB_ORDER_ASC; + } } - tsBufFlush(output1); - tsBufFlush(output2); - - tsBufDestroy(pSupporter1->pTSBuf); - pSupporter1->pTSBuf = NULL; - tsBufDestroy(pSupporter2->pTSBuf); - pSupporter2->pTSBuf = NULL; + for (int32_t i = 0; i < joinNum; ++i) { + tsBufFlush(ctxlist[i].res); + + tsBufDestroy(ctxlist[i].p->pTSBuf); + ctxlist[i].p->pTSBuf = NULL; + } TSKEY et = taosGetTimestampUs(); - tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", - pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfGroups, win->skey, win->ekey, - tsBufGetNumOfGroup(output1), et - st); - return output1->numOfTotal; + for (int32_t i = 0; i < joinNum; ++i) { + tscDebug("%p sub:%p tblidx:%d, input:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " + "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", + pSql, pSql->pSubs[i], i, ctxlist[i].numOfInput, ctxlist[i].res->numOfTotal, ctxlist[i].res->numOfGroups, win->skey, win->ekey, + tsBufGetNumOfGroup(ctxlist[i].res), et - st); + } + + return ctxlist[0].res->numOfTotal; } + // todo handle failed to create sub query SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, int32_t index) { SJoinSupporter* pSupporter = calloc(1, sizeof(SJoinSupporter)); @@ -367,10 +551,6 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { // scan all subquery, if one sub query has only ts, ignore it tscDebug("%p start to launch secondary subqueries, %d out of %d needs to query", pSql, numOfSub, pSql->subState.numOfSub); - //the subqueries that do not actually launch the secondary query to virtual node is set as completed. - SSubqueryState* pState = &pSql->subState; - pState->numOfRemain = numOfSub; - bool success = true; for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { @@ -394,7 +574,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pSubQueryInfo->tsBuf = NULL; // free result for async object will also free sqlObj - assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one resutl columns + assert(tscSqlExprNumOfExprs(pSubQueryInfo) == 1); // ts_comp query only requires one result columns taos_free_result(pPrevSub); SSqlObj *pNew = createSubqueryObj(pSql, (int16_t) i, tscJoinQueryCallback, pSupporter, TSDB_SQL_SELECT, NULL); @@ -403,6 +583,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { success = false; break; } + tscClearSubqueryInfo(&pNew->cmd); pSql->pSubs[i] = pNew; @@ -447,12 +628,13 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0); int16_t funcId = pExpr->functionId; + // add the invisible timestamp column if ((pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) || (funcId != TSDB_FUNC_TS && funcId != TSDB_FUNC_TS_DUMMY && funcId != TSDB_FUNC_PRJ)) { int16_t functionId = tscIsProjectionQuery(pQueryInfo)? TSDB_FUNC_PRJ : TSDB_FUNC_TS; - tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL); tscPrintSelectClause(pNew, 0); tscFieldInfoUpdateOffset(pQueryInfo); @@ -480,10 +662,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { } } + subquerySetState(pPrevSub, &pSql->subState, i, 0); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), - numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); + numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); } //prepare the subqueries object failed, abort @@ -517,22 +701,28 @@ void freeJoinSubqueryObj(SSqlObj* pSql) { SJoinSupporter* p = pSub->param; tscDestroyJoinSupporter(p); - if (pSub->res.code == TSDB_CODE_SUCCESS) { - taos_free_result(pSub); - } + taos_free_result(pSub); + pSql->pSubs[i] = NULL; } + if (pSql->subState.states) { + pthread_mutex_destroy(&pSql->subState.mutex); + } + + tfree(pSql->subState.states); + + pSql->subState.numOfSub = 0; } -static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { - assert(pSqlObj->subState.numOfRemain > 0); - - if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) { - tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code)); +static int32_t quitAllSubquery(SSqlObj* pSqlSub, SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { + if (subAndCheckDone(pSqlSub, pSqlObj, pSupporter->subqueryIndex)) { + tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code)); freeJoinSubqueryObj(pSqlObj); + return 0; } + return 1; //tscDestroyJoinSupporter(pSupporter); } @@ -615,16 +805,20 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr prev = tt; } - pTableMetaInfo->pVgroupTables = result; pTableMetaInfo->vgroupIndex = 0; + + if (taosArrayGetSize(result) <= 0) { + pTableMetaInfo->pVgroupTables = NULL; + taosArrayDestroy(result); + } else { + pTableMetaInfo->pVgroupTables = result; - if (taosArrayGetSize(result) > 0) { SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1); tscDebug("%p vgId:%d, tables:%"PRIzu, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList)); } } -static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { +static void issueTsCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { SSqlCmd* pCmd = &pSql->cmd; tscClearSubqueryInfo(pCmd); tscFreeSqlResult(pSql); @@ -644,7 +838,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; SColumnIndex index = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pQueryInfo, 0, TSDB_FUNC_TS_COMP, &index, &colSchema, TSDB_COL_NORMAL); // set the tags value for ts_comp function if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { @@ -674,7 +868,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* "%p subquery:%p tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, ts_comp query to retrieve timestamps, " "numOfExpr:%" PRIzu ", colList:%" PRIzu ", numOfOutputFields:%d, name:%s", pParent, pSql, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pQueryInfo->type, - tscSqlExprNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); + tscSqlExprNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); tscProcessSql(pSql); } @@ -695,76 +889,218 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq return true; } -static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) { - SJoinSupporter* p1 = pParentSql->pSubs[0]->param; - SJoinSupporter* p2 = pParentSql->pSubs[1]->param; - - tscDebug("%p all subquery retrieve complete, do tags match, %d, %d", pParentSql, p1->num, p2->num); - - // sort according to the tag value - qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar); - qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar); +static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray* resList) { + int16_t joinNum = pParentSql->subState.numOfSub; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); + SJoinSupporter* p0 = pParentSql->pSubs[0]->param; + SMergeCtx ctxlist[TSDB_MAX_JOIN_TABLE_NUM] = {{0}}; + SMergeCtx* ctxStack[TSDB_MAX_JOIN_TABLE_NUM] = {0}; + + // int16_t for padding + int32_t size = p0->tagSize - sizeof(int16_t); SSchema* pColSchema = tscGetColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + + tscDebug("%p all subquery retrieve complete, do tags match", pParentSql); - // int16_t for padding - int32_t size = p1->tagSize - sizeof(int16_t); - *s1 = taosArrayInit(p1->num, size); - *s2 = taosArrayInit(p2->num, size); - - if (!(checkForDuplicateTagVal(pColSchema, p1, pParentSql) && checkForDuplicateTagVal(pColSchema, p2, pParentSql))) { - return TSDB_CODE_QRY_DUP_JOIN_KEY; - } - - int32_t i = 0, j = 0; - while(i < p1->num && j < p2->num) { - STidTags* pp1 = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); - STidTags* pp2 = (STidTags*) varDataVal(p2->pIdTagList + j * p2->tagSize); - assert(pp1->tid != 0 && pp2->tid != 0); - - int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes); - if (ret == 0) { - tscDebug("%p tag matched, vgId:%d, val:%d, tid:%d, uid:%"PRIu64", tid:%d, uid:%"PRIu64, pParentSql, pp1->vgId, - *(int*) pp1->tag, pp1->tid, pp1->uid, pp2->tid, pp2->uid); - - taosArrayPush(*s1, pp1); - taosArrayPush(*s2, pp2); - j++; - i++; - } else if (ret > 0) { - j++; - } else { - i++; + for (int32_t i = 0; i < joinNum; i++) { + SJoinSupporter* p = pParentSql->pSubs[i]->param; + + ctxlist[i].p = p; + ctxlist[i].res = taosArrayInit(p->num, size); + + tscDebug("Join %d - num:%d", i, p->num); + + // sort according to the tag valu + qsort(p->pIdTagList, p->num, p->tagSize, tagValCompar); + + if (!checkForDuplicateTagVal(pColSchema, p, pParentSql)) { + for (int32_t j = 0; j <= i; j++) { + taosArrayDestroy(ctxlist[j].res); + } + return TSDB_CODE_QRY_DUP_JOIN_KEY; } } - // reorganize the tid-tag value according to both the vgroup id and tag values - // sort according to the tag value - size_t t1 = taosArrayGetSize(*s1); - size_t t2 = taosArrayGetSize(*s2); + int32_t slot = 0; + size_t tableNum = 0; + int16_t* tableMIdx = 0; + int32_t equalNum = 0; + int32_t stackidx = 0; + int32_t mergeDone = 0; + SMergeCtx* ctx = NULL; + SMergeCtx* pctx = NULL; + STidTags* cur = NULL; + STidTags* prev = NULL; + SArray* tagCond = NULL; + + for (int16_t tidx = 0; tidx < joinNum; tidx++) { + pctx = &ctxlist[tidx]; + if (pctx->compared) { + continue; + } + + assert(pctx->idx == 0 && taosArrayGetSize(pctx->res) == 0); - qsort((*s1)->pData, t1, size, tidTagsCompar); - qsort((*s2)->pData, t2, size, tidTagsCompar); + tagCond = pQueryInfo->tagCond.joinInfo.joinTables[tidx]->tagJoin; -#if 0 - for(int32_t k = 0; k < t1; ++k) { - STidTags* p = (*s1)->pData + size * k; - printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data); - } + tableNum = taosArrayGetSize(tagCond); + assert(tableNum >= 2); + + for (int32_t i = 0; i < tableNum; ++i) { + tableMIdx = taosArrayGet(tagCond, i); + SMergeCtx* tctx = &ctxlist[*tableMIdx]; + tctx->compared = 1; + } + + for (int32_t i = 0; i < tableNum; ++i) { + tableMIdx = taosArrayGet(tagCond, i); + SMergeCtx* tctx = &ctxlist[*tableMIdx]; + if (tctx->p->num <= 0 || tctx->p->pIdTagList == NULL) { + mergeDone = 1; + break; + } + } + + if (mergeDone) { + mergeDone = 0; + continue; + } + + tableMIdx = taosArrayGet(tagCond, slot); + + pctx = &ctxlist[*tableMIdx]; + + prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); + + ctxStack[stackidx++] = pctx; + + tableMIdx = taosArrayGet(tagCond, ++slot); + + equalNum = 1; + + while (1) { + ctx = &ctxlist[*tableMIdx]; + + cur = (STidTags*) varDataVal(ctx->p->pIdTagList + ctx->idx * ctx->p->tagSize); + + assert(cur->tid != 0 && prev->tid != 0); + + ctxStack[stackidx++] = ctx; + + int32_t ret = doCompare(prev->tag, cur->tag, pColSchema->type, pColSchema->bytes); + if (ret == 0) { + if (++equalNum < tableNum) { + prev = cur; + pctx = ctx; + + if (++slot >= tableNum) { + slot = 0; + } + + tableMIdx = taosArrayGet(tagCond, slot); + continue; + } + + tscDebug("%p tag matched, vgId:%d, val:%d, tid:%d, uid:%"PRIu64", tid:%d, uid:%"PRIu64, pParentSql, prev->vgId, + *(int*) prev->tag, prev->tid, prev->uid, cur->tid, cur->uid); + + assert(stackidx == tableNum); + + for (int32_t i = 0; i < stackidx; ++i) { + SMergeCtx* tctx = ctxStack[i]; + prev = (STidTags*) varDataVal(tctx->p->pIdTagList + tctx->idx * tctx->p->tagSize); + + taosArrayPush(tctx->res, prev); + } + + for (int32_t i = 0; i < stackidx; ++i) { + SMergeCtx* tctx = ctxStack[i]; + + if (++tctx->idx >= tctx->p->num) { + mergeDone = 1; + break; + } + } + + if (mergeDone) { + break; + } + + stackidx = 0; + equalNum = 1; + + prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); + + ctxStack[stackidx++] = pctx; + } else if (ret > 0) { + stackidx--; + + if (++ctx->idx >= ctx->p->num) { + break; + } + } else { + stackidx--; + + for (int32_t i = 0; i < stackidx; ++i) { + SMergeCtx* tctx = ctxStack[i]; + if (++tctx->idx >= tctx->p->num) { + mergeDone = 1; + break; + } + } + + if (mergeDone) { + break; + } - for(int32_t k = 0; k < t1; ++k) { - STidTags* p = (*s2)->pData + size * k; - printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data); + stackidx = 0; + equalNum = 1; + + prev = (STidTags*) varDataVal(pctx->p->pIdTagList + pctx->idx * pctx->p->tagSize); + ctxStack[stackidx++] = pctx; + } + + } + + slot = 0; + mergeDone = 0; + stackidx = 0; } -#endif - tscDebug("%p tags match complete, result: %"PRIzu", %"PRIzu, pParentSql, t1, t2); + for (int32_t i = 0; i < joinNum; ++i) { + // reorganize the tid-tag value according to both the vgroup id and tag values + // sort according to the tag value + size_t num = taosArrayGetSize(ctxlist[i].res); + + qsort((ctxlist[i].res)->pData, num, size, tidTagsCompar); + + taosArrayPush(resList, &ctxlist[i].res); + + tscDebug("%p tags match complete, result num: %"PRIzu, pParentSql, num); + } + return TSDB_CODE_SUCCESS; } +bool emptyTagList(SArray* resList, int32_t size) { + size_t rsize = taosArrayGetSize(resList); + if (rsize != size) { + return true; + } + + for (int32_t i = 0; i < size; ++i) { + SArray** s = taosArrayGet(resList, i); + if (taosArrayGetSize(*s) <= 0) { + return true; + } + } + + return false; +} + static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) { SJoinSupporter* pSupporter = (SJoinSupporter*)param; @@ -775,8 +1111,21 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow SSqlRes* pRes = &pSql->res; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + // todo, the type may not include TSDB_QUERY_TYPE_TAG_FILTER_QUERY assert(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)); + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + + tscAsyncResultOnError(pParentSql); + + return; + } + // check for the error code firstly if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // todo retry if other subqueries are not failed @@ -785,7 +1134,9 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); pParentSql->res.code = numOfRows; - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } tscAsyncResultOnError(pParentSql); return; @@ -802,7 +1153,9 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p failed to malloc memory", pSql); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } tscAsyncResultOnError(pParentSql); return; @@ -844,23 +1197,24 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // no data exists in next vnode, mark the query completed // only when there is no subquery exits any more, proceeds to get the intersect of the tuple sets. - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { + tscDebug("%p tagRetrieve:%p,%d completed, total:%d", pParentSql, tres, pSupporter->subqueryIndex, pParentSql->subState.numOfSub); return; - } + } + + SArray* resList = taosArrayInit(pParentSql->subState.numOfSub, sizeof(SArray *)); - SArray *s1 = NULL, *s2 = NULL; - int32_t code = getIntersectionOfTableTuple(pQueryInfo, pParentSql, &s1, &s2); + int32_t code = getIntersectionOfTableTuple(pQueryInfo, pParentSql, resList); if (code != TSDB_CODE_SUCCESS) { freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; tscAsyncResultOnError(pParentSql); - taosArrayDestroy(s1); - taosArrayDestroy(s2); + taosArrayDestroy(resList); return; } - if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return. + if (emptyTagList(resList, pParentSql->subState.numOfSub)) { // no results,return. assert(pParentSql->fp != tscJoinQueryCallback); tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql); @@ -872,35 +1226,34 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { - // proceed to for ts_comp query - SSqlCmd* pSubCmd1 = &pParentSql->pSubs[0]->cmd; - SSqlCmd* pSubCmd2 = &pParentSql->pSubs[1]->cmd; - - SQueryInfo* pQueryInfo1 = tscGetQueryInfoDetail(pSubCmd1, 0); - STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo1, 0); - tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo1, s1); - - SQueryInfo* pQueryInfo2 = tscGetQueryInfoDetail(pSubCmd2, 0); - STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0); - tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2); - - SSqlObj* psub1 = pParentSql->pSubs[0]; - ((SJoinSupporter*)psub1->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo1->pVgroupTables); - - SSqlObj* psub2 = pParentSql->pSubs[1]; - ((SJoinSupporter*)psub2->param)->pVgroupTables = tscVgroupTableInfoClone(pTableMetaInfo2->pVgroupTables); - - pParentSql->subState.numOfSub = 2; - pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; - for (int32_t m = 0; m < pParentSql->subState.numOfSub; ++m) { - SSqlObj* sub = pParentSql->pSubs[m]; - issueTSCompQuery(sub, sub->param, pParentSql); + // proceed to for ts_comp query + SSqlCmd* pSubCmd = &pParentSql->pSubs[m]->cmd; + SArray** s = taosArrayGet(resList, m); + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo, *s); + + SSqlObj* psub = pParentSql->pSubs[m]; + ((SJoinSupporter*)psub->param)->pVgroupTables = tscVgroupTableInfoDup(pTableMetaInfo->pVgroupTables); + + memset(pParentSql->subState.states, 0, sizeof(pParentSql->subState.states[0]) * pParentSql->subState.numOfSub); + tscDebug("%p reset all sub states to 0", pParentSql); + + issueTsCompQuery(psub, psub->param, pParentSql); + } + } + + size_t rsize = taosArrayGetSize(resList); + for (int32_t i = 0; i < rsize; ++i) { + SArray** s = taosArrayGet(resList, i); + if (*s) { + taosArrayDestroy(*s); } } - taosArrayDestroy(s1); - taosArrayDestroy(s2); + taosArrayDestroy(resList); } static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRows) { @@ -915,6 +1268,17 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert(!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)); + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + if (quitAllSubquery(pSql, pParentSql, pSupporter)){ + return; + } + + tscAsyncResultOnError(pParentSql); + + return; + } + // check for the error code firstly if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { // todo retry if other subqueries are not failed yet @@ -922,7 +1286,9 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex); pParentSql->res.code = numOfRows; - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)){ + return; + } tscAsyncResultOnError(pParentSql); return; @@ -930,14 +1296,16 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow if (numOfRows > 0) { // write the compressed timestamp to disk file if(pSupporter->f == NULL) { - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); if (pSupporter->f == NULL) { tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno)); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } tscAsyncResultOnError(pParentSql); @@ -954,8 +1322,9 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscError("%p invalid ts comp file from vnode, abort subquery, file size:%d", pSql, numOfRows); pParentSql->res.code = TAOS_SYSTEM_ERROR(errno); - - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)){ + return; + } tscAsyncResultOnError(pParentSql); @@ -974,7 +1343,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // continue to retrieve ts-comp data from vnode if (!pRes->completed) { taosGetTmpfilePath("ts-join", pSupporter->path); - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); pRes->row = pRes->numOfRows; taos_fetch_rows_a(tres, tsCompRetrieveCallback, param); @@ -1000,7 +1369,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow taosGetTmpfilePath("ts-join", pSupporter->path); // TODO check for failure - pSupporter->f = fopen(pSupporter->path, "w"); + pSupporter->f = fopen(pSupporter->path, "wb"); pRes->row = pRes->numOfRows; // set the callback function @@ -1009,18 +1378,14 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow return; } - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { return; - } + } tscDebug("%p all subquery retrieve ts complete, do ts block intersect", pParentSql); - // proceeds to launched secondary query to retrieve final data - SJoinSupporter* p1 = pParentSql->pSubs[0]->param; - SJoinSupporter* p2 = pParentSql->pSubs[1]->param; - STimeWindow win = TSWINDOW_INITIALIZER; - int64_t num = doTSBlockIntersect(pParentSql, p1, p2, &win); + int64_t num = doTSBlockIntersect(pParentSql, &win); if (num <= 0) { // no result during ts intersect tscDebug("%p no results generated in ts intersection, free all sub SqlObj and quit", pParentSql); freeJoinSubqueryObj(pParentSql); @@ -1049,6 +1414,19 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR SSqlRes* pRes = &pSql->res; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + if (pParentSql->res.code != TSDB_CODE_SUCCESS) { + tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + + tscAsyncResultOnError(pParentSql); + + return; + } + + if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { assert(numOfRows == taos_errno(pSql)); @@ -1088,9 +1466,8 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } } - assert(pState->numOfRemain > 0); - if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { - tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub); + if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { + tscDebug("%p sub:%p,%d completed, total:%d", pParentSql, tres, pSupporter->subqueryIndex, pState->numOfSub); return; } @@ -1205,15 +1582,16 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { } } - // get the number of subquery that need to retrieve the next vnode. + if (orderedPrjQuery) { for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { - pSql->subState.numOfRemain++; + subquerySetState(pSub, &pSql->subState, i, 0); } } } + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; @@ -1270,7 +1648,19 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { // retrieve data from current vnode. tscDebug("%p retrieve data from %d subqueries", pSql, numOfFetch); SJoinSupporter* pSupporter = NULL; - pSql->subState.numOfRemain = numOfFetch; + + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSql1 = pSql->pSubs[i]; + if (pSql1 == NULL) { + continue; + } + + SSqlRes* pRes1 = &pSql1->res; + + if (pRes1->row >= pRes1->numOfRows) { + subquerySetState(pSql1, &pSql->subState, i, 0); + } + } for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSql1 = pSql->pSubs[i]; @@ -1353,7 +1743,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } // restore the offset value for super table query in case of final result. - tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscRestoreFuncForSTableQuery(pQueryInfo); tscFieldInfoUpdateOffset(pQueryInfo); } @@ -1372,7 +1762,10 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { // retrieve actual query results from vnode during the second stage join subquery if (pParentSql->res.code != TSDB_CODE_SUCCESS) { tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code); - quitAllSubquery(pParentSql, pSupporter); + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + tscAsyncResultOnError(pParentSql); return; @@ -1384,7 +1777,11 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { tscError("%p abort query, code:%s, global code:%s", pSql, tstrerror(code), tstrerror(pParentSql->res.code)); pParentSql->res.code = code; - quitAllSubquery(pParentSql, pSupporter); + + if (quitAllSubquery(pSql, pParentSql, pSupporter)) { + return; + } + tscAsyncResultOnError(pParentSql); return; @@ -1408,9 +1805,9 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { // In case of consequence query from other vnode, do not wait for other query response here. if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) { - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) { return; - } + } } tscSetupOutputColumnIndex(pParentSql); @@ -1422,6 +1819,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data pSql->cmd.command = TSDB_SQL_FETCH; + tscProcessSql(pSql); } else { // first retrieve from vnode during the secondary stage sub-query // set the command flag must be after the semaphore been correctly set. @@ -1442,7 +1840,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter SSqlCmd * pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pSql->res.qhandle = 0x1; + pSql->res.qId = 0x1; assert(pSql->res.numOfRows == 0); if (pSql->pSubs == NULL) { @@ -1457,8 +1855,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pSql->pSubs[pSql->subState.numOfRemain++] = pNew; - assert(pSql->subState.numOfRemain <= pSql->subState.numOfSub); + pSql->pSubs[tableIndex] = pNew; if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { addGroupInfoForSubquery(pSql, pNew, 0, tableIndex); @@ -1498,6 +1895,8 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter pNewQueryInfo->limit.limit = -1; pNewQueryInfo->limit.offset = 0; + pNewQueryInfo->order.orderColId = INT32_MIN; + // backup the data and clear it in the sqlcmd object memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr)); @@ -1527,18 +1926,18 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter // set get tags query type TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); - tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG); + tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TID_TAG, &colIndex, &s1, TSDB_COL_TAG); size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscDebug( "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), " "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, tagIndex:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, colIndex.columnIndex, pNewQueryInfo->pTableMetaInfo[0]->name); + numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, colIndex.columnIndex, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); } else { SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = 1}; SColumnIndex colIndex = {0, PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscAddSpecialColumnForSelect(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL); + tscAddFuncInSelectClause(pNewQueryInfo, 0, TSDB_FUNC_TS_COMP, &colIndex, &colSchema, TSDB_COL_NORMAL); // set the tags value for ts_comp function SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); @@ -1569,7 +1968,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%u, transfer to ts_comp query to retrieve timestamps, " "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); + numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pNewQueryInfo->pTableMetaInfo[0]->name)); } } else { assert(0); @@ -1590,6 +1989,19 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { int32_t code = TSDB_CODE_SUCCESS; pSql->subState.numOfSub = pQueryInfo->numOfTables; + if (pSql->subState.states == NULL) { + pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(*pSql->subState.states)); + if (pSql->subState.states == NULL) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + pthread_mutex_init(&pSql->subState.mutex, NULL); + } + + memset(pSql->subState.states, 0, sizeof(*pSql->subState.states) * pSql->subState.numOfSub); + tscDebug("%p reset all sub states to 0", pSql); + bool hasEmptySub = false; tscDebug("%p start subquery, total:%d", pSql, pQueryInfo->numOfTables); @@ -1622,14 +2034,25 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { pSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; (*pSql->fp)(pSql->param, pSql, 0); } else { + int fail = 0; for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; + if (fail) { + (*pSub->fp)(pSub->param, pSub, 0); + continue; + } + if ((code = tscProcessSql(pSub)) != TSDB_CODE_SUCCESS) { - pSql->subState.numOfRemain = i - 1; // the already sent request will continue and do not go to the error process routine - break; + pRes->code = code; + (*pSub->fp)(pSub->param, pSub, 0); + fail = 1; } } + if(fail) { + return; + } + pSql->cmd.command = TSDB_SQL_TABLE_JOIN_RETRIEVE; } @@ -1673,7 +2096,334 @@ void tscUnlockByThread(int64_t *lockedBy) { } } +typedef struct SFirstRoundQuerySup { + SSqlObj *pParent; + int32_t numOfRows; + SArray *pColsInfo; + int32_t tagLen; + STColumn *pTagCols; + SArray *pResult; // SArray + int64_t interval; + char* buf; + int32_t bufLen; +} SFirstRoundQuerySup; + +void doAppendData(SInterResult* pInterResult, TAOS_ROW row, int32_t numOfCols, SQueryInfo* pQueryInfo) { + TSKEY key = INT64_MIN; + for(int32_t i = 0; i < numOfCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (TSDB_COL_IS_TAG(pExpr->colInfo.flag) || pExpr->functionId == TSDB_FUNC_PRJ) { + continue; + } + + if (pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + key = *(TSKEY*) row[i]; + continue; + } + + double v = 0; + if (row[i] != NULL) { + v = *(double*) row[i]; + } else { + SET_DOUBLE_NULL(&v); + } + + int32_t id = pExpr->colInfo.colId; + int32_t numOfQueriedCols = (int32_t) taosArrayGetSize(pInterResult->pResult); + + SArray* p = NULL; + for(int32_t j = 0; j < numOfQueriedCols; ++j) { + SStddevInterResult* pColRes = taosArrayGet(pInterResult->pResult, j); + if (pColRes->colId == id) { + p = pColRes->pResult; + break; + } + } + + if (p && taosArrayGetSize(p) > 0) { + SResPair *l = taosArrayGetLast(p); + if (l->key == key && key == INT64_MIN) { + continue; + } + } + //append a new column + if (p == NULL) { + SStddevInterResult t = {.colId = id, .pResult = taosArrayInit(10, sizeof(SResPair)),}; + taosArrayPush(pInterResult->pResult, &t); + p = t.pResult; + } + + SResPair pair = {.avg = v, .key = key}; + taosArrayPush(p, &pair); + } +} + +static void destroySup(SFirstRoundQuerySup* pSup) { + taosArrayDestroyEx(pSup->pResult, freeInterResult); + taosArrayDestroy(pSup->pColsInfo); + tfree(pSup); +} + +void tscFirstRoundRetrieveCallback(void* param, TAOS_RES* tres, int numOfRows) { + SSqlObj* pSql = (SSqlObj*)tres; + SSqlRes* pRes = &pSql->res; + + SFirstRoundQuerySup* pSup = param; + + SSqlObj* pParent = pSup->pParent; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + + int32_t code = taos_errno(pSql); + if (code != TSDB_CODE_SUCCESS) { + destroySup(pSup); + taos_free_result(pSql); + pParent->res.code = code; + tscAsyncResultOnError(pParent); + return; + } + + if (numOfRows > 0) { // the number is not correct for group by column in super table query + TAOS_ROW row = NULL; + int32_t numOfCols = taos_field_count(tres); + + if (pSup->tagLen == 0) { // no tags, all rows belong to one group + SInterResult interResult = {.tags = NULL, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; + taosArrayPush(pSup->pResult, &interResult); + + while ((row = taos_fetch_row(tres)) != NULL) { + doAppendData(&interResult, row, numOfCols, pQueryInfo); + pSup->numOfRows += 1; + } + } else { // tagLen > 0 + char* p = calloc(1, pSup->tagLen); + + while ((row = taos_fetch_row(tres)) != NULL) { + int32_t* length = taos_fetch_lengths(tres); + memset(p, 0, pSup->tagLen); + + int32_t offset = 0; + for (int32_t i = 0; i < numOfCols && offset < pSup->tagLen; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + + // tag or group by column + if (TSDB_COL_IS_TAG(pExpr->colInfo.flag) || pExpr->functionId == TSDB_FUNC_PRJ) { + if (row[i] == NULL) { + setNull(p + offset, pExpr->resType, pExpr->resBytes); + } else { + memcpy(p + offset, row[i], length[i]); + } + offset += pExpr->resBytes; + } + } + + assert(offset == pSup->tagLen); + size_t size = taosArrayGetSize(pSup->pResult); + + if (size > 0) { + SInterResult* pInterResult = taosArrayGetLast(pSup->pResult); + if (memcmp(pInterResult->tags, p, pSup->tagLen) == 0) { // belongs to the same group + doAppendData(pInterResult, row, numOfCols, pQueryInfo); + } else { + char* tags = malloc( pSup->tagLen); + memcpy(tags, p, pSup->tagLen); + + SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; + taosArrayPush(pSup->pResult, &interResult); + doAppendData(&interResult, row, numOfCols, pQueryInfo); + } + } else { + char* tags = malloc(pSup->tagLen); + memcpy(tags, p, pSup->tagLen); + + SInterResult interResult = {.tags = tags, .pResult = taosArrayInit(4, sizeof(SStddevInterResult))}; + taosArrayPush(pSup->pResult, &interResult); + doAppendData(&interResult, row, numOfCols, pQueryInfo); + } + + pSup->numOfRows += 1; + } + + tfree(p); + } + } + + if (!pRes->completed) { + taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param); + return; + } + + // set the parameters for the second round query process + SSqlCmd *pPCmd = &pParent->cmd; + SQueryInfo *pQueryInfo1 = tscGetQueryInfoDetail(pPCmd, 0); + int32_t resRows = pSup->numOfRows; + + if (pSup->numOfRows > 0) { + SBufferWriter bw = tbufInitWriter(NULL, false); + interResToBinary(&bw, pSup->pResult, pSup->tagLen); + + pQueryInfo1->bufLen = (int32_t) tbufTell(&bw); + pQueryInfo1->buf = tbufGetData(&bw, true); + + // set the serialized binary string as the parameter of arithmetic expression + tbufCloseWriter(&bw); + } + + taosArrayDestroyEx(pSup->pResult, freeInterResult); + taosArrayDestroy(pSup->pColsInfo); + tfree(pSup); + + taos_free_result(pSql); + + if (resRows == 0) { + pParent->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + (*pParent->fp)(pParent->param, pParent, 0); + return; + } + + pQueryInfo1->round = 1; + tscDoQuery(pParent); +} + +void tscFirstRoundCallback(void* param, TAOS_RES* tres, int code) { + SFirstRoundQuerySup* pSup = (SFirstRoundQuerySup*) param; + + SSqlObj* pSql = (SSqlObj*) tres; + int32_t c = taos_errno(pSql); + + if (c != TSDB_CODE_SUCCESS) { + SSqlObj* parent = pSup->pParent; + + destroySup(pSup); + taos_free_result(pSql); + parent->res.code = code; + tscAsyncResultOnError(parent); + return; + } + + taos_fetch_rows_a(tres, tscFirstRoundRetrieveCallback, param); +} + +int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + STableMetaInfo* pTableMetaInfo1 = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); + + SFirstRoundQuerySup *pSup = calloc(1, sizeof(SFirstRoundQuerySup)); + + pSup->pParent = pSql; + pSup->interval = pQueryInfo->interval.interval; + pSup->pResult = taosArrayInit(6, sizeof(SStddevInterResult)); + pSup->pColsInfo = taosArrayInit(6, sizeof(int16_t)); // result column id + + SSqlObj *pNew = createSubqueryObj(pSql, 0, tscFirstRoundCallback, pSup, TSDB_SQL_SELECT, NULL); + SSqlCmd *pCmd = &pNew->cmd; + + tscClearSubqueryInfo(pCmd); + tscFreeSqlResult(pSql); + + SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + assert(pQueryInfo->numOfTables == 1); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); + + tscInitQueryInfo(pNewQueryInfo); + pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; + if (pQueryInfo->groupbyExpr.columnInfo != NULL) { + pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); + if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + } + + if (tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + pNewQueryInfo->interval = pQueryInfo->interval; + + pCmd->command = TSDB_SQL_SELECT; + pNew->fp = tscFirstRoundCallback; + + int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + + int32_t index = 0; + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_TS && pQueryInfo->interval.interval > 0) { + taosArrayPush(pSup->pColsInfo, &pExpr->resColId); + + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->colInfo.colId); + + SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TS, &colIndex, schema, TSDB_COL_NORMAL); + p->resColId = pExpr->resColId; // update the result column id + } else if (pExpr->functionId == TSDB_FUNC_STDDEV_DST) { + taosArrayPush(pSup->pColsInfo, &pExpr->resColId); + + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->colInfo.colIndex}; + SSchema schema = {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; + tstrncpy(schema.name, pExpr->aliasName, tListLen(schema.name)); + + SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_AVG, &colIndex, &schema, TSDB_COL_NORMAL); + p->resColId = pExpr->resColId; // update the result column id + } else if (pExpr->functionId == TSDB_FUNC_TAG) { + pSup->tagLen += pExpr->resBytes; + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pExpr->colInfo.colIndex}; + + SSchema* schema = NULL; + if (pExpr->colInfo.colId != TSDB_TBNAME_COLUMN_INDEX) { + schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->colInfo.colId); + } else { + schema = tGetTbnameColumnSchema(); + } + + SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_TAG, &colIndex, schema, TSDB_COL_TAG); + p->resColId = pExpr->resColId; + } else if (pExpr->functionId == TSDB_FUNC_PRJ) { + int32_t num = (int32_t) taosArrayGetSize(pNewQueryInfo->groupbyExpr.columnInfo); + for(int32_t k = 0; k < num; ++k) { + SColIndex* pIndex = taosArrayGet(pNewQueryInfo->groupbyExpr.columnInfo, k); + if (pExpr->colInfo.colId == pIndex->colId) { + pSup->tagLen += pExpr->resBytes; + taosArrayPush(pSup->pColsInfo, &pExpr->resColId); + + SColumnIndex colIndex = {.tableIndex = 0, .columnIndex = pIndex->colIndex}; + SSchema* schema = tscGetColumnSchemaById(pTableMetaInfo1->pTableMeta, pExpr->colInfo.colId); + + //doLimitOutputNormalColOfGroupby + SSqlExpr* p = tscAddFuncInSelectClause(pNewQueryInfo, index++, TSDB_FUNC_PRJ, &colIndex, schema, TSDB_COL_NORMAL); + p->numOfParams = 1; + p->param[0].i64 = 1; + p->param[0].nType = TSDB_DATA_TYPE_INT; + p->resColId = pExpr->resColId; // update the result column id + } + } + } + } + + SColumnIndex columnIndex = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscInsertPrimaryTsSourceColumn(pNewQueryInfo, &columnIndex); + + tscTansformFuncForSTableQuery(pNewQueryInfo); + + tscDebug( + "%p first round subquery:%p tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, query to retrieve timestamps, " + "numOfExpr:%" PRIzu ", colList:%d, numOfOutputFields:%d, name:%s", + pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pNewQueryInfo->type, + tscSqlExprNumOfExprs(pNewQueryInfo), index+1, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pTableMetaInfo->name)); + + tscHandleMasterSTableQuery(pNew); + return TSDB_CODE_SUCCESS; + + _error: + destroySup(pSup); + taos_free_result(pNew); + pSql->res.code = terrno; + tscAsyncResultOnError(pSql); + return terrno; +} int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; @@ -1685,12 +2435,12 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { return pRes->code; } - tExtMemBuffer ** pMemoryBuf = NULL; + tExtMemBuffer **pMemoryBuf = NULL; tOrderDescriptor *pDesc = NULL; SColumnModel *pModel = NULL; SColumnModel *pFinalModel = NULL; - pRes->qhandle = 0x1; // hack the qhandle check + pRes->qId = 0x1; // hack the qhandle check const uint32_t nBufferSize = (1u << 16u); // 64KB @@ -1714,11 +2464,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tfree(pMemoryBuf); return ret; } - - pSql->pSubs = calloc(pState->numOfSub, POINTER_BYTES); tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub); - + pSql->pSubs = calloc(pState->numOfSub, POINTER_BYTES); if (pSql->pSubs == NULL) { tfree(pSql->pSubs); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1728,7 +2476,21 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { return ret; } - pState->numOfRemain = pState->numOfSub; + if (pState->states == NULL) { + pState->states = calloc(pState->numOfSub, sizeof(*pState->states)); + if (pState->states == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + tscAsyncResultOnError(pSql); + tfree(pMemoryBuf); + return ret; + } + + pthread_mutex_init(&pState->mutex, NULL); + } + + memset(pState->states, 0, sizeof(*pState->states) * pState->numOfSub); + tscDebug("%p reset all sub states to 0", pSql); + pRes->code = TSDB_CODE_SUCCESS; int32_t i = 0; @@ -1829,7 +2591,24 @@ static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES * current query failed, and the retry count is less than the available * count, retry query clear previous retrieved data, then launch a new sub query */ -static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, int32_t code) { +static int32_t tscReissueSubquery(SRetrieveSupport *oriTrs, SSqlObj *pSql, int32_t code, int32_t *sent) { + *sent = 0; + + SRetrieveSupport *trsupport = malloc(sizeof(SRetrieveSupport)); + if (trsupport == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + memcpy(trsupport, oriTrs, sizeof(*trsupport)); + + const uint32_t nBufferSize = (1u << 16u); // 64KB + trsupport->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); + if (trsupport->localBuffer == NULL) { + tscError("%p failed to malloc buffer for local buffer, reason:%s", pSql, strerror(errno)); + tfree(trsupport); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + SSqlObj *pParentSql = trsupport->pParentSql; int32_t subqueryIndex = trsupport->subqueryIndex; @@ -1846,21 +2625,28 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", - trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); + oriTrs->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, oriTrs->subqueryIndex); pParentSql->res.code = terrno; - trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + oriTrs->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; + tfree(trsupport); return pParentSql->res.code; } int32_t ret = tscProcessSql(pNew); + *sent = 1; + // if failed to process sql, let following code handle the pSql if (ret == TSDB_CODE_SUCCESS) { + tscFreeRetrieveSup(pSql); taos_free_result(pSql); return ret; - } else { + } else { + pParentSql->pSubs[trsupport->subqueryIndex] = pSql; + tscFreeRetrieveSup(pNew); + taos_free_result(pNew); return ret; } } @@ -1877,7 +2663,6 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO assert(pSql != NULL); SSubqueryState* pState = &pParentSql->subState; - assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0); // retrieved in subquery failed. OR query cancelled in retrieve phase. if (taos_errno(pSql) == TSDB_CODE_SUCCESS && pParentSql->res.code != TSDB_CODE_SUCCESS) { @@ -1898,7 +2683,10 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO subqueryIndex, tstrerror(pParentSql->res.code)); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { - if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { + int32_t sent = 0; + + tscReissueSubquery(trsupport, pSql, numOfRows, &sent); + if (sent) { return; } } else { // reach the maximum retry count, abort @@ -1908,14 +2696,12 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO } } - int32_t remain = -1; - if ((remain = atomic_sub_fetch_32(&pState->numOfRemain, 1)) > 0) { - tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex, - pState->numOfSub - remain); + if (!subAndCheckDone(pSql, pParentSql, subqueryIndex)) { + tscDebug("%p sub:%p,%d freed, not finished, total:%d", pParentSql, pSql, trsupport->subqueryIndex, pState->numOfSub); tscFreeRetrieveSup(pSql); return; - } + } // all subqueries are failed tscError("%p retrieve from %d vnode(s) completed,code:%s.FAILED.", pParentSql, pState->numOfSub, @@ -1980,14 +2766,12 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p return; } - int32_t remain = -1; - if ((remain = atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1)) > 0) { - tscDebug("%p sub:%p orderOfSub:%d freed, finished subqueries:%d", pParentSql, pSql, trsupport->subqueryIndex, - pState->numOfSub - remain); + if (!subAndCheckDone(pSql, pParentSql, idx)) { + tscDebug("%p sub:%p orderOfSub:%d freed, not finished", pParentSql, pSql, trsupport->subqueryIndex); tscFreeRetrieveSup(pSql); return; - } + } // all sub-queries are returned, start to local merge process pDesc->pColumnModel->capacity = trsupport->pExtMemBuffer[idx]->numOfElemsPerPage; @@ -1998,7 +2782,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p SQueryInfo *pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); tscClearInterpInfo(pPQueryInfo); - tscCreateLocalReducer(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, trsupport->pFFColModel, pParentSql); + tscCreateLocalMerger(trsupport->pExtMemBuffer, pState->numOfSub, pDesc, trsupport->pFinalColModel, trsupport->pFFColModel, pParentSql); tscDebug("%p build loser tree completed", pParentSql); pParentSql->res.precision = pSql->res.precision; @@ -2024,7 +2808,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR SRetrieveSupport *trsupport = (SRetrieveSupport *)param; if (pSql->param == NULL || param == NULL) { tscDebug("%p already freed in dnodecallback", pSql); - assert(pSql->res.code == TSDB_CODE_TSC_QUERY_CANCELLED); return; } @@ -2033,7 +2816,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR SSqlObj * pParentSql = trsupport->pParentSql; SSubqueryState* pState = &pParentSql->subState; - assert(pState->numOfRemain <= pState->numOfSub && pState->numOfRemain >= 0); STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); SVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; @@ -2057,7 +2839,10 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); - if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { + int32_t sent = 0; + + tscReissueSubquery(trsupport, pSql, numOfRows, &sent); + if (sent) { return; } } else { @@ -2125,12 +2910,17 @@ static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsuppo SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; + + // clear the limit/offset info, since it should not be sent to vnode to be executed. + pQueryInfo->limit.limit = -1; + pQueryInfo->limit.offset = 0; + assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->subState.numOfSub); // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); pTableMetaInfo->vgroupIndex = trsupport->subqueryIndex; - + pSql->pSubs[trsupport->subqueryIndex] = pNew; } @@ -2179,7 +2969,11 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); - if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { + + int32_t sent = 0; + + tscReissueSubquery(trsupport, pSql, code, &sent); + if (sent) { return; } } else { @@ -2194,7 +2988,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { tscDebug("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); - if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode + if (pSql->res.qId == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode tscRetrieveFromDnodeCallBack(param, pSql, 0); } else { taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); @@ -2239,7 +3033,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) // record the total inserted rows if (numOfRows > 0) { - pParentObj->res.numOfRows += numOfRows; + atomic_add_fetch_32(&pParentObj->res.numOfRows, numOfRows); } if (taos_errno(tres) != TSDB_CODE_SUCCESS) { @@ -2254,7 +3048,8 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) } } - if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) { + if (!subAndCheckDone(tres, pParentObj, pSupporter->index)) { + tscDebug("%p insert:%p,%d completed, total:%d", pParentObj, tres, pSupporter->index, pParentObj->subState.numOfSub); return; } @@ -2283,10 +3078,12 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) numOfFailed += 1; // clean up tableMeta in cache - tscFreeQueryInfo(&pSql->cmd); + tscFreeQueryInfo(&pSql->cmd, false); SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pParentObj->cmd, pSql->cmd.clauseIndex, 0); - tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); + tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); + + subquerySetState(pSql, &pParentObj->subState, i, 0); tscDebug("%p, failed sub:%d, %p", pParentObj, i, pSql); } @@ -2297,14 +3094,14 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) tscDebug("%p cleanup %d tableMeta in hashTable", pParentObj, pParentObj->cmd.numOfTables); for(int32_t i = 0; i < pParentObj->cmd.numOfTables; ++i) { - char* name = pParentObj->cmd.pTableNameList[i]; + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(pParentObj->cmd.pTableNameList[i], name); taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } pParentObj->cmd.parseFinished = false; - pParentObj->subState.numOfRemain = numOfFailed; - tscResetSqlCmdObj(&pParentObj->cmd); + tscResetSqlCmd(&pParentObj->cmd, false); // in case of insert, redo parsing the sql string and build new submit data block for two reasons: // 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly. @@ -2378,7 +3175,19 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { // the number of already initialized subqueries int32_t numOfSub = 0; - pSql->subState.numOfRemain = pSql->subState.numOfSub; + if (pSql->subState.states == NULL) { + pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(*pSql->subState.states)); + if (pSql->subState.states == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + pthread_mutex_init(&pSql->subState.mutex, NULL); + } + + memset(pSql->subState.states, 0, sizeof(*pSql->subState.states) * pSql->subState.numOfSub); + tscDebug("%p reset all sub states to 0", pSql); + pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES); if (pSql->pSubs == NULL) { goto _error; @@ -2553,7 +3362,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { return; } - tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscRestoreFuncForSTableQuery(pQueryInfo); } assert (pRes->row >= pRes->numOfRows); @@ -2565,30 +3374,6 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { } } -static UNUSED_FUNC void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { - SSqlRes *pRes = &pSql->res; - - if (pRes->tsrow[columnIndex] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) { - // convert unicode to native code in a temporary buffer extra one byte for terminated symbol - if (pRes->buffer[columnIndex] == NULL) { - pRes->buffer[columnIndex] = malloc(pField->bytes + TSDB_NCHAR_SIZE); - } - - /* string terminated char for binary data*/ - memset(pRes->buffer[columnIndex], 0, pField->bytes + TSDB_NCHAR_SIZE); - - int32_t length = taosUcs4ToMbs(pRes->tsrow[columnIndex], pRes->length[columnIndex], pRes->buffer[columnIndex]); - if ( length >= 0 ) { - pRes->tsrow[columnIndex] = (unsigned char*)pRes->buffer[columnIndex]; - pRes->length[columnIndex] = length; - } else { - tscError("%p charset:%s to %s. val:%s convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)pRes->tsrow[columnIndex]); - pRes->tsrow[columnIndex] = NULL; - pRes->length[columnIndex] = 0; - } - } -} - char *getArithmeticInputSrc(void *param, const char *name, int32_t colId) { SArithmeticSupport *pSupport = (SArithmeticSupport *) param; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index dd0f248b85452342fb567357ba9346ab71754dfc..bd79f81846cbfd730fd9b3134b89ba0b328472a1 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -18,7 +18,6 @@ #include "tref.h" #include "trpc.h" #include "tnote.h" -#include "tsystem.h" #include "ttimer.h" #include "tutil.h" #include "tsched.h" @@ -43,41 +42,75 @@ void *tscTmr; void *tscQhandle; int32_t tscRefId = -1; int32_t tscNumOfObj = 0; // number of sqlObj in current process. - static void *tscCheckDiskUsageTmr; +void *tscRpcCache; // cache to keep rpc obj +int32_t tscNumOfThreads = 1; // num of rpc threads +static pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently static pthread_once_t tscinit = PTHREAD_ONCE_INIT; +static volatile int tscInitRes = 0; -void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { +void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) { taosGetDisk(); - taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); + taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); +} +void tscFreeRpcObj(void *param) { + assert(param); + SRpcObj *pRpcObj = (SRpcObj *)(param); + tscDebug("free rpcObj:%p and free pDnodeConn: %p", pRpcObj, pRpcObj->pDnodeConn); + rpcClose(pRpcObj->pDnodeConn); } -int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeConn) { - SRpcInit rpcInit; - - if (*pDnodeConn == NULL) { - memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = 0; - rpcInit.label = "TSC"; - rpcInit.numOfThreads = 1; // every DB connection has only one thread - rpcInit.cfp = tscProcessMsgFromServer; - rpcInit.sessions = tsMaxConnections; - rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char *)user; - rpcInit.idleTime = 2000; - rpcInit.ckey = "key"; - rpcInit.spi = 1; - rpcInit.secret = (char *)secretEncrypt; - - *pDnodeConn = rpcOpen(&rpcInit); - if (*pDnodeConn == NULL) { - tscError("failed to init connection to TDengine"); - return -1; - } else { - tscDebug("dnodeConn:%p is created, user:%s", *pDnodeConn, user); - } +void tscReleaseRpc(void *param) { + if (param == NULL) { + return; } + pthread_mutex_lock(&rpcObjMutex); + taosCacheRelease(tscRpcCache, (void *)¶m, false); + pthread_mutex_unlock(&rpcObjMutex); +} + +int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncrypt, void **ppRpcObj) { + pthread_mutex_lock(&rpcObjMutex); + + SRpcObj *pRpcObj = (SRpcObj *)taosCacheAcquireByKey(tscRpcCache, key, strlen(key)); + if (pRpcObj != NULL) { + *ppRpcObj = pRpcObj; + pthread_mutex_unlock(&rpcObjMutex); + return 0; + } + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "TSC"; + rpcInit.numOfThreads = tscNumOfThreads; + rpcInit.cfp = tscProcessMsgFromServer; + rpcInit.sessions = tsMaxConnections; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.user = (char *)user; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.ckey = "key"; + rpcInit.spi = 1; + rpcInit.secret = (char *)secretEncrypt; + + SRpcObj rpcObj; + memset(&rpcObj, 0, sizeof(rpcObj)); + strncpy(rpcObj.key, key, strlen(key)); + rpcObj.pDnodeConn = rpcOpen(&rpcInit); + if (rpcObj.pDnodeConn == NULL) { + pthread_mutex_unlock(&rpcObjMutex); + tscError("failed to init connection to TDengine"); + return -1; + } + pRpcObj = taosCachePut(tscRpcCache, rpcObj.key, strlen(rpcObj.key), &rpcObj, sizeof(rpcObj), 1000*5); + if (pRpcObj == NULL) { + rpcClose(rpcObj.pDnodeConn); + pthread_mutex_unlock(&rpcObjMutex); + return -1; + } + + *ppRpcObj = pRpcObj; + pthread_mutex_unlock(&rpcObjMutex); return 0; } @@ -105,7 +138,11 @@ void taos_init_imp(void) { } taosReadGlobalCfg(); - taosCheckGlobalCfg(); + if (taosCheckGlobalCfg()) { + tscInitRes = -1; + return; + } + taosInitNotes(); rpcInit(); @@ -118,20 +155,20 @@ void taos_init_imp(void) { int queueSize = tsMaxConnections*2; double factor = (tscEmbedded == 0)? 2.0:4.0; - int32_t tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); + tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); if (tscNumOfThreads < 2) { tscNumOfThreads = 2; } - tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc"); if (NULL == tscQhandle) { tscError("failed to init scheduler"); + tscInitRes = -1; return; } tscTmr = taosTmrInit(tsMaxConnections * 2, 200, 60000, "TSC"); if(0 == tscEmbedded){ - taosTmrReset(tscCheckDiskUsage, 10, NULL, tscTmr, &tscCheckDiskUsageTmr); + taosTmrReset(tscCheckDiskUsage, 20 * 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); } if (tscTableMetaInfo == NULL) { @@ -140,6 +177,10 @@ void taos_init_imp(void) { tscTableMetaInfo = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tscDebug("TableMeta:%p", tscTableMetaInfo); } + + int refreshTime = 5; + tscRpcCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, true, tscFreeRpcObj, "rpcObj"); + pthread_mutex_init(&rpcObjMutex, NULL); tscRefId = taosOpenRef(200, tscCloseTscObj); @@ -150,7 +191,7 @@ void taos_init_imp(void) { tscDebug("client is initialized successfully"); } -void taos_init() { pthread_once(&tscinit, taos_init_imp); } +int taos_init() { pthread_once(&tscinit, taos_init_imp); return tscInitRes;} // this function may be called by user or system, or by both simultaneously. void taos_cleanup(void) { @@ -179,11 +220,19 @@ void taos_cleanup(void) { taosCloseRef(id); taosCleanupKeywordsTable(); - taosCloseLog(); + + p = tscRpcCache; + tscRpcCache = NULL; + + if (p != NULL) { + taosCacheCleanup(p); + pthread_mutex_destroy(&rpcObjMutex); + } if (tscEmbedded == 0) { rpcCleanup(); - } + taosCloseLog(); + }; p = tscTmr; tscTmr = NULL; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9a8aa917e70f978c2d5c4f0038223479eb10bbd3..57b57dba5a2982fc04447b6c3f60a67744935e7a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -16,7 +16,7 @@ #include "tscUtil.h" #include "hash.h" #include "os.h" -#include "qAst.h" +#include "texpr.h" #include "taosmsg.h" #include "tkey.h" #include "tmd5.h" @@ -30,8 +30,16 @@ #include "ttokendef.h" static void freeQueryInfoImpl(SQueryInfo* pQueryInfo); -static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo); +static void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta); +static void tscStrToLower(char *str, int32_t n) { + if (str == NULL || n <= 0) { return;} + for (int32_t i = 0; i < n; i++) { + if (str[i] >= 'A' && str[i] <= 'Z') { + str[i] -= ('A' - 'a'); + } + } +} SCond* tsGetSTableQueryCond(STagCond* pTagCond, uint64_t uid) { if (pTagCond->pCond == NULL) { return NULL; @@ -89,19 +97,20 @@ bool tscQueryTags(SQueryInfo* pQueryInfo) { return true; } -// todo refactor, extract methods and move the common module -void tscGetDBInfoFromTableFullName(char* tableId, char* db) { - char* st = strstr(tableId, TS_PATH_DELIMITER); - if (st != NULL) { - char* end = strstr(st + 1, TS_PATH_DELIMITER); - if (end != NULL) { - memcpy(db, tableId, (end - tableId)); - db[end - tableId] = 0; - return; +bool tscQueryBlockInfo(SQueryInfo* pQueryInfo) { + int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + + for (int32_t i = 0; i < numOfCols; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + int32_t functId = pExpr->functionId; + + // "select count(tbname)" query + if (functId == TSDB_FUNC_BLKINFO) { + return true; } } - db[0] = 0; + return false; } bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { @@ -114,11 +123,6 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { return false; } - // for select query super table, the super table vgroup list can not be null in any cases. - // if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - // assert(pTableMetaInfo->vgroupList != NULL); - // } - if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { return false; } @@ -235,6 +239,21 @@ bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo) { return false; } +bool tscGroupbyColumn(SQueryInfo* pQueryInfo) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); + + SSqlGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr; + for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { + SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); + if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) { // group by normal columns + return true; + } + } + + return false; +} + bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { @@ -290,7 +309,7 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { int32_t offset = 0; - for (int32_t i = 0; i < pRes->numOfCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); pRes->urow[i] = pRes->data + offset * pRes->numOfRows; @@ -379,7 +398,7 @@ static void tscDestroyResPointerInfo(SSqlRes* pRes) { pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free } -void tscFreeQueryInfo(SSqlCmd* pCmd) { +void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { if (pCmd == NULL || pCmd->numOfClause == 0) { return; } @@ -388,7 +407,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, i); freeQueryInfoImpl(pQueryInfo); - clearAllTableMetaInfo(pQueryInfo); + clearAllTableMetaInfo(pQueryInfo, removeMeta); tfree(pQueryInfo); } @@ -396,7 +415,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd) { tfree(pCmd->pQueryInfo); } -void tscResetSqlCmdObj(SSqlCmd* pCmd) { +void tscResetSqlCmd(SSqlCmd* pCmd, bool removeMeta) { pCmd->command = 0; pCmd->numOfCols = 0; pCmd->count = 0; @@ -414,13 +433,13 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd) { pCmd->numOfTables = 0; tfree(pCmd->pTableNameList); - pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList); + pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, removeMeta); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - tscFreeQueryInfo(pCmd); + tscFreeQueryInfo(pCmd, removeMeta); } void tscFreeSqlResult(SSqlObj* pSql) { - tscDestroyLocalReducer(pSql); + tscDestroyLocalMerger(pSql); SSqlRes* pRes = &pSql->res; tscDestroyResPointerInfo(pRes); @@ -428,7 +447,7 @@ void tscFreeSqlResult(SSqlObj* pSql) { memset(&pSql->res, 0, sizeof(SSqlRes)); } -static void tscFreeSubobj(SSqlObj* pSql) { +void tscFreeSubobj(SSqlObj* pSql) { if (pSql->subState.numOfSub == 0) { return; } @@ -441,6 +460,12 @@ static void tscFreeSubobj(SSqlObj* pSql) { pSql->pSubs[i] = NULL; } + if (pSql->subState.states) { + pthread_mutex_destroy(&pSql->subState.mutex); + } + + tfree(pSql->subState.states); + pSql->subState.numOfSub = 0; } @@ -456,7 +481,6 @@ void tscFreeRegisteredSqlObj(void *pSql) { SSqlObj* p = *(SSqlObj**)pSql; STscObj* pTscObj = p->pTscObj; - assert(RID_VALID(p->self)); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfObj, 1); @@ -510,7 +534,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->self = 0; tscFreeSqlResult(pSql); - tscResetSqlCmdObj(pCmd); + tscResetSqlCmd(pCmd, false); tfree(pCmd->tagData.data); pCmd->tagData.dataLen = 0; @@ -524,7 +548,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { free(pSql); } -void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { +void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { if (pDataBlock == NULL) { return; } @@ -537,6 +561,13 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { tfree(pDataBlock->pTableMeta); } + if (removeMeta) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pDataBlock->tableName, name); + + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + } + tfree(pDataBlock); } @@ -572,21 +603,21 @@ void* tscDestroyBlockArrayList(SArray* pDataBlockList) { size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { void* d = taosArrayGetP(pDataBlockList, i); - tscDestroyDataBlock(d); + tscDestroyDataBlock(d, false); } taosArrayDestroy(pDataBlockList); return NULL; } -void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable) { +void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) { if (pBlockHashTable == NULL) { return NULL; } STableDataBlocks** p = taosHashIterate(pBlockHashTable, NULL); while(p) { - tscDestroyDataBlock(*p); + tscDestroyDataBlock(*p, removeMeta); p = taosHashIterate(pBlockHashTable, p); } @@ -603,17 +634,16 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { assert(pCmd->numOfClause == 1); STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + // todo refactor // set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) { - tstrncpy(pTableMetaInfo->name, pDataBlock->tableName, sizeof(pTableMetaInfo->name)); + tNameAssign(&pTableMetaInfo->name, &pDataBlock->tableName); if (pTableMetaInfo->pTableMeta != NULL) { tfree(pTableMetaInfo->pTableMeta); } - pTableMetaInfo->pTableMeta = tscTableMetaClone(pDataBlock->pTableMeta); - } else { - assert(strncmp(pTableMetaInfo->name, pDataBlock->tableName, tListLen(pDataBlock->tableName)) == 0); + pTableMetaInfo->pTableMeta = tscTableMetaDup(pDataBlock->pTableMeta); } /* @@ -648,7 +678,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { * @param dataBlocks * @return */ -int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name, +int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { @@ -676,27 +706,26 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff dataBuf->size = startOffset; dataBuf->tsSource = -1; - tstrncpy(dataBuf->tableName, name, sizeof(dataBuf->tableName)); + tNameAssign(&dataBuf->tableName, name); //Here we keep the tableMeta to avoid it to be remove by other threads. - dataBuf->pTableMeta = tscTableMetaClone(pTableMeta); + dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; return TSDB_CODE_SUCCESS; } -int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, const char* tableId, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks, SArray* pBlockList) { +int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, + SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { *dataBlocks = *t1; } if (*dataBlocks == NULL) { - int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, tableId, pTableMeta, dataBlocks); + int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -785,22 +814,28 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { return result; } -static void extractTableNameList(SSqlCmd* pCmd) { +static void extractTableNameList(SSqlCmd* pCmd, bool freeBlockMap) { pCmd->numOfTables = (int32_t) taosHashGetSize(pCmd->pTableBlockHashList); - pCmd->pTableNameList = calloc(pCmd->numOfTables, POINTER_BYTES); + if (pCmd->pTableNameList == NULL) { + pCmd->pTableNameList = calloc(pCmd->numOfTables, POINTER_BYTES); + } else { + memset(pCmd->pTableNameList, 0, pCmd->numOfTables * POINTER_BYTES); + } STableDataBlocks **p1 = taosHashIterate(pCmd->pTableBlockHashList, NULL); int32_t i = 0; while(p1) { STableDataBlocks* pBlocks = *p1; - pCmd->pTableNameList[i++] = strndup(pBlocks->tableName, TSDB_TABLE_FNAME_LEN); + pCmd->pTableNameList[i++] = tNameDup(&pBlocks->tableName); p1 = taosHashIterate(pCmd->pTableBlockHashList, p1); } - pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList); + if (freeBlockMap) { + pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, false); + } } -int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { +int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap) { const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg); SSqlCmd* pCmd = &pSql->cmd; @@ -816,7 +851,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { STableDataBlocks* dataBuf = NULL; int32_t ret = tscGetDataBlockFromList(pVnodeDataBlockHashList, pOneTableBlock->vgId, TSDB_PAYLOAD_SIZE, - INSERT_HEAD_SIZE, 0, pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); + INSERT_HEAD_SIZE, 0, &pOneTableBlock->tableName, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList); if (ret != TSDB_CODE_SUCCESS) { tscError("%p failed to prepare the data block buffer for merging table data, code:%d", pSql, ret); taosHashCleanup(pVnodeDataBlockHashList); @@ -849,8 +884,8 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { tscSortRemoveDataBlockDupRows(pOneTableBlock); char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1); - - tscDebug("%p name:%s, sid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, pOneTableBlock->tableName, + + tscDebug("%p name:%s, name:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pSql, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey)); int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta); @@ -880,7 +915,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql) { pOneTableBlock = *p; } - extractTableNameList(pCmd); + extractTableNameList(pCmd, freeBlockMap); // free the table data blocks; pCmd->pDataBlocks = pVnodeDataBlockList; @@ -896,16 +931,10 @@ void tscCloseTscObj(void *param) { pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); - void* p = pObj->pDnodeConn; - if (pObj->pDnodeConn != NULL) { - rpcClose(pObj->pDnodeConn); - pObj->pDnodeConn = NULL; - } - tfree(pObj->tscCorMgmtEpSet); + tscReleaseRpc(pObj->pRpcObj); pthread_mutex_destroy(&pObj->mutex); - tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p); tfree(pObj); } @@ -1052,7 +1081,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { SInternalField* pInfo = taosArrayGet(pFieldInfo->internalField, i); if (pInfo->pArithExprInfo != NULL) { - tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); + tExprTreeDestroy(pInfo->pArithExprInfo->pExpr, NULL); SSqlFuncMsg* pFuncMsg = &pInfo->pArithExprInfo->base; for(int32_t j = 0; j < pFuncMsg->numOfParams; ++j) { @@ -1071,7 +1100,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { memset(pFieldInfo, 0, sizeof(SFieldInfo)); } -static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, +static SSqlExpr* doCreateSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, int16_t size, int16_t resColId, int16_t interSize, int32_t colType) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); @@ -1085,6 +1114,8 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; + } else if (pColIndex->columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) { + pExpr->colInfo.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { pExpr->colInfo.colId = pColIndex->columnIndex; } else { @@ -1122,14 +1153,14 @@ SSqlExpr* tscSqlExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functi return tscSqlExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); } - SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); + SSqlExpr* pExpr = doCreateSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); taosArrayInsert(pQueryInfo->exprList, index, &pExpr); return pExpr; } SSqlExpr* tscSqlExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { - SSqlExpr* pExpr = doBuildSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); + int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) { + SSqlExpr* pExpr = doCreateSqlExpr(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol); taosArrayPush(pQueryInfo->exprList, &pExpr); return pExpr; } @@ -1153,6 +1184,22 @@ SSqlExpr* tscSqlExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functi return pExpr; } +bool tscMultiRoundQuery(SQueryInfo* pQueryInfo, int32_t index) { + if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) { + return false; + } + + int32_t numOfExprs = (int32_t) tscSqlExprNumOfExprs(pQueryInfo); + for(int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr->functionId == TSDB_FUNC_STDDEV_DST) { + return true; + } + } + + return false; +} + size_t tscSqlExprNumOfExprs(SQueryInfo* pQueryInfo) { return taosArrayGetSize(pQueryInfo->exprList); } @@ -1232,6 +1279,34 @@ int32_t tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepco return 0; } +bool tscColumnExists(SArray* pColumnList, SColumnIndex* pColIndex) { + // ignore the tbname columnIndex to be inserted into source list + if (pColIndex->columnIndex < 0) { + return false; + } + + size_t numOfCols = taosArrayGetSize(pColumnList); + int16_t col = pColIndex->columnIndex; + + int32_t i = 0; + while (i < numOfCols) { + SColumn* pCol = taosArrayGetP(pColumnList, i); + if ((pCol->colIndex.columnIndex != col) || (pCol->colIndex.tableIndex != pColIndex->tableIndex)) { + ++i; + continue; + } else { + break; + } + } + + if (i >= numOfCols || numOfCols == 0) { + return false; + } + + return true; +} + + SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { // ignore the tbname columnIndex to be inserted into source list if (pColIndex->columnIndex < 0) { @@ -1298,7 +1373,7 @@ SColumn* tscColumnClone(const SColumn* src) { dst->colIndex = src->colIndex; dst->numOfFilters = src->numOfFilters; - dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); + dst->filterInfo = tFilterInfoDup(src->filterInfo, src->numOfFilters); return dst; } @@ -1416,9 +1491,11 @@ int32_t tscValidateName(SStrToken* pToken) { char* sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true); if (sep == NULL) { // single part if (pToken->type == TK_STRING) { - strdequote(pToken->z); - pToken->n = (uint32_t)strtrim(pToken->z); - + + tscDequoteAndTrimToken(pToken); + tscStrToLower(pToken->z, pToken->n); + //pToken->n = (uint32_t)strtrim(pToken->z); + int len = tSQLGetToken(pToken->z, &pToken->type); // single token, validate it @@ -1470,7 +1547,7 @@ int32_t tscValidateName(SStrToken* pToken) { if (pToken->type == TK_STRING && validateQuoteToken(pToken) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } - + // re-build the whole name string if (pStr[firstPartLen] == TS_PATH_DELIMITER[0]) { // first part do not have quote do nothing @@ -1482,6 +1559,8 @@ int32_t tscValidateName(SStrToken* pToken) { } pToken->n += (firstPartLen + sizeof(TS_PATH_DELIMITER[0])); pToken->z = pStr; + + tscStrToLower(pToken->z,pToken->n); } return TSDB_CODE_SUCCESS; @@ -1501,7 +1580,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || colId == TSDB_BLOCK_DIST_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { return true; } @@ -1532,7 +1611,25 @@ int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) { dest->tbnameCond.uid = src->tbnameCond.uid; dest->tbnameCond.len = src->tbnameCond.len; - memcpy(&dest->joinInfo, &src->joinInfo, sizeof(SJoinInfo)); + dest->joinInfo.hasJoin = src->joinInfo.hasJoin; + + for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { + if (src->joinInfo.joinTables[i]) { + dest->joinInfo.joinTables[i] = calloc(1, sizeof(SJoinNode)); + + memcpy(dest->joinInfo.joinTables[i], src->joinInfo.joinTables[i], sizeof(SJoinNode)); + + if (src->joinInfo.joinTables[i]->tsJoin) { + dest->joinInfo.joinTables[i]->tsJoin = taosArrayDup(src->joinInfo.joinTables[i]->tsJoin); + } + + if (src->joinInfo.joinTables[i]->tagJoin) { + dest->joinInfo.joinTables[i]->tagJoin = taosArrayDup(src->joinInfo.joinTables[i]->tagJoin); + } + } + } + + dest->relType = src->relType; if (src->pCond == NULL) { @@ -1578,6 +1675,23 @@ void tscTagCondRelease(STagCond* pTagCond) { taosArrayDestroy(pTagCond->pCond); } + for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) { + SJoinNode *node = pTagCond->joinInfo.joinTables[i]; + if (node == NULL) { + continue; + } + + if (node->tsJoin != NULL) { + taosArrayDestroy(node->tsJoin); + } + + if (node->tagJoin != NULL) { + taosArrayDestroy(node->tagJoin); + } + + tfree(node); + } + memset(pTagCond, 0, sizeof(STagCond)); } @@ -1702,10 +1816,15 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField)); assert(pQueryInfo->exprList == NULL); - pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); - pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; - pQueryInfo->resColumnId= -1000; + pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; + pQueryInfo->resColumnId = -1000; + pQueryInfo->limit.limit = -1; + pQueryInfo->limit.offset = 0; + + pQueryInfo->slimit.limit = -1; + pQueryInfo->slimit.offset = 0; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { @@ -1753,6 +1872,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); tfree(pQueryInfo->fillVal); + tfree(pQueryInfo->buf); } void tscClearSubqueryInfo(SSqlCmd* pCmd) { @@ -1804,10 +1924,10 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) { info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); } - info->itemList = taosArrayClone(pInfo->itemList); + info->itemList = taosArrayDup(pInfo->itemList); } -SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { +SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) { if (pVgroupTables == NULL) { return NULL; } @@ -1826,10 +1946,17 @@ SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { return pa; } -void clearAllTableMetaInfo(SQueryInfo* pQueryInfo) { +void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, bool removeMeta) { for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + if (removeMeta) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + + taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); + } + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); tscClearTableMetaInfo(pTableMetaInfo); free(pTableMetaInfo); @@ -1838,7 +1965,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo) { tfree(pQueryInfo->pTableMetaInfo); } -STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, +STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, SName* name, STableMeta* pTableMeta, SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { @@ -1856,7 +1983,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; if (name != NULL) { - tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name)); + tNameAssign(&pTableMetaInfo->name, name); } pTableMetaInfo->pTableMeta = pTableMeta; @@ -1875,7 +2002,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } - pTableMetaInfo->pVgroupTables = tscVgroupTableInfoClone(pVgroupTables); + pTableMetaInfo->pVgroupTables = tscVgroupTableInfoDup(pVgroupTables); pQueryInfo->numOfTables += 1; return pTableMetaInfo; @@ -1915,7 +2042,7 @@ void registerSqlObj(SSqlObj* pSql) { tscDebug("%p new SqlObj from %p, total in tscObj:%d, total:%d", pSql, pSql->pTscObj, num, total); } -SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) { +SSqlObj* createSimpleSubObj(SSqlObj* pSql, __async_cb_func_t fp, void* param, int32_t cmd) { SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { tscError("%p new subquery failed, tableIndex:%d", pSql, 0); @@ -1938,6 +2065,10 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm } if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { +#ifdef __APPLE__ + // to satisfy later tsem_destroy in taos_free_result + tsem_init(&pNew->rspSem, 0, 0); +#endif // __APPLE__ tscFreeSqlObj(pNew); return NULL; } @@ -1953,7 +2084,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); + tscAddTableMetaInfo(pQueryInfo, &pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); registerSqlObj(pNew); return pNew; @@ -2000,7 +2131,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pNewQueryInfo, int64_t ui tscFieldInfoUpdateOffset(pNewQueryInfo); } -SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { +SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t fp, void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); @@ -2016,7 +2147,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->signature = pNew; pNew->sqlstr = strdup(pSql->sqlstr); - SSqlCmd* pnCmd = &pNew->cmd; + SSqlCmd* pnCmd = &pNew->cmd; memcpy(pnCmd, pCmd, sizeof(SSqlCmd)); pnCmd->command = cmd; @@ -2032,6 +2163,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->parseFinished = 1; pnCmd->pTableNameList = NULL; pnCmd->pTableBlockHashList = NULL; + pnCmd->tagData.data = NULL; + pnCmd->tagData.dataLen = 0; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -2055,10 +2188,21 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit; pNewQueryInfo->numOfTables = 0; pNewQueryInfo->pTableMetaInfo = NULL; - + pNewQueryInfo->bufLen = pQueryInfo->bufLen; + + pNewQueryInfo->buf = malloc(pQueryInfo->bufLen); + if (pNewQueryInfo->buf == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + + if (pQueryInfo->bufLen > 0) { + memcpy(pNewQueryInfo->buf, pQueryInfo->buf, pQueryInfo->bufLen); + } + pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { - pNewQueryInfo->groupbyExpr.columnInfo = taosArrayClone(pQueryInfo->groupbyExpr.columnInfo); + pNewQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo); if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; @@ -2109,27 +2253,30 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->param = param; pNew->maxRetry = TSDB_MAX_REPLICA; - char* name = pTableMetaInfo->name; STableMetaInfo* pFinalInfo = NULL; if (pPrevSql == NULL) { - STableMeta* pTableMeta = tscTableMetaClone(pTableMetaInfo->pTableMeta); + STableMeta* pTableMeta = tscTableMetaDup(pTableMetaInfo->pTableMeta); assert(pTableMeta != NULL); - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); - - STableMeta* pPrevTableMeta = tscTableMetaClone(pPrevInfo->pTableMeta); + if (pPrevInfo->pTableMeta && pPrevInfo->pTableMeta->tableType < 0) { + terrno = TSDB_CODE_TSC_APP_ERROR; + goto _error; + } + + STableMeta* pPrevTableMeta = tscTableMetaDup(pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, &pTableMetaInfo->name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } // this case cannot be happened if (pFinalInfo->pTableMeta == NULL) { - tscError("%p new subquery failed since no tableMeta, name:%s", pSql, name); + tscError("%p new subquery failed since no tableMeta, name:%s", pSql, tNameGetTableName(&pTableMetaInfo->name)); if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); @@ -2154,7 +2301,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu "," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), - size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey, + size, pNewQueryInfo->fieldsInfo.numOfOutput, tNameGetTableName(&pFinalInfo->name), pNewQueryInfo->window.skey, pNewQueryInfo->window.ekey, pNewQueryInfo->order.order, pNewQueryInfo->limit.limit); tscPrintSelectClause(pNew, 0); @@ -2191,12 +2338,12 @@ void tscDoQuery(SSqlObj* pSql) { } if (pCmd->dataSourceType == DATA_FROM_DATA_FILE) { - tscProcessMultiVnodesImportFromFile(pSql); + tscImportDataFromFile(pSql); } else { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); uint16_t type = pQueryInfo->type; - if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) { // multi-vnodes insertion + if (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_INSERT)) { // multi-vnodes insertion tscHandleMultivnodeInsert(pSql); return; } @@ -2218,6 +2365,9 @@ void tscDoQuery(SSqlObj* pSql) { } } + return; + } else if (tscMultiRoundQuery(pQueryInfo, 0) && pQueryInfo->round == 0) { + tscHandleFirstRoundStableQuery(pSql); // todo lock? return; } else if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { // super table query tscLockByThread(&pSql->squeryLock); @@ -2231,16 +2381,21 @@ void tscDoQuery(SSqlObj* pSql) { } int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) { - if (pTagCond->joinInfo.left.uid == uid) { - return pTagCond->joinInfo.left.tagColId; - } else if (pTagCond->joinInfo.right.uid == uid) { - return pTagCond->joinInfo.right.tagColId; - } else { - assert(0); - return -1; + int32_t i = 0; + while (i < TSDB_MAX_JOIN_TABLE_NUM) { + SJoinNode* node = pTagCond->joinInfo.joinTables[i]; + if (node && node->uid == uid) { + return node->tagColId; + } + + i++; } + + assert(0); + return -1; } + int16_t tscGetTagColIndexById(STableMeta* pTableMeta, int16_t colId) { int32_t numOfTags = tscGetNumOfTags(pTableMeta); @@ -2266,6 +2421,35 @@ bool tscIsUpdateQuery(SSqlObj* pSql) { return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) || TSDB_SQL_USE_DB == pCmd->command); } +char* tscGetSqlStr(SSqlObj* pSql) { + if (pSql == NULL || pSql->signature != pSql) { + return NULL; + } + + return pSql->sqlstr; +} + +bool tscIsQueryWithLimit(SSqlObj* pSql) { + if (pSql == NULL || pSql->signature != pSql) { + return false; + } + + SSqlCmd* pCmd = &pSql->cmd; + for (int32_t i = 0; i < pCmd->numOfClause; ++i) { + SQueryInfo* pqi = tscGetQueryInfoDetailSafely(pCmd, i); + if (pqi == NULL) { + continue; + } + + if (pqi->limit.limit > 0) { + return true; + } + } + + return false; +} + + int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql) { const char* msgFormat1 = "syntax error near \'%s\'"; const char* msgFormat2 = "syntax error near \'%s\' (%s)"; @@ -2291,7 +2475,6 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s } return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) { @@ -2506,7 +2689,7 @@ bool tscSetSqlOwner(SSqlObj* pSql) { SSqlRes* pRes = &pSql->res; // set the sql object owner - uint64_t threadId = taosGetSelfPthreadId(); + int64_t threadId = taosGetSelfPthreadId(); if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { pRes->code = TSDB_CODE_QRY_IN_EXEC; return false; @@ -2516,7 +2699,6 @@ bool tscSetSqlOwner(SSqlObj* pSql) { } void tscClearSqlOwner(SSqlObj* pSql) { - assert(taosCheckPthreadValid(pSql->owner)); atomic_store_64(&pSql->owner, 0); } @@ -2643,7 +2825,11 @@ STableMeta* createSuperTableMeta(STableMetaMsg* pChild) { uint32_t tscGetTableMetaSize(STableMeta* pTableMeta) { assert(pTableMeta != NULL); - int32_t totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + int32_t totalCols = 0; + if (pTableMeta->tableInfo.numOfColumns >= 0 && pTableMeta->tableInfo.numOfTags >= 0) { + totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; + } + return sizeof(STableMeta) + totalCols * sizeof(SSchema); } @@ -2689,7 +2875,7 @@ uint32_t tscGetTableMetaMaxSize() { return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema); } -STableMeta* tscTableMetaClone(STableMeta* pTableMeta) { +STableMeta* tscTableMetaDup(STableMeta* pTableMeta) { assert(pTableMeta != NULL); uint32_t size = tscGetTableMetaSize(pTableMeta); STableMeta* p = calloc(1, size); diff --git a/src/client/tests/CMakeLists.txt b/src/client/tests/CMakeLists.txt index 4ea0b80bf7c1c870532f1bc3cac313c43f1a57f0..f07af85e255eaa5e285d9a4ce0853251e0fdaa21 100644 --- a/src/client/tests/CMakeLists.txt +++ b/src/client/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) diff --git a/src/client/tests/resultFieldTest.cpp b/src/client/tests/cliTest.cpp similarity index 58% rename from src/client/tests/resultFieldTest.cpp rename to src/client/tests/cliTest.cpp index c917f0ebafa02480c372e8d7bb611588bad8e54a..30f248b5418b54b1be26dfcf15348b03fd70af4d 100644 --- a/src/client/tests/resultFieldTest.cpp +++ b/src/client/tests/cliTest.cpp @@ -2,15 +2,117 @@ #include #include "taos.h" +#include "tglobal.h" namespace { static int64_t start_ts = 1433955661000; + +void stmtInsertTest() { + TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); + if (conn == NULL) { + printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); + exit(-1); + } + + TAOS_RES* res = taos_query(conn, "use test"); + taos_free_result(res); + + const char* sql = "insert into t1 values(?, ?, ?, ?)"; + TAOS_STMT* stmt = taos_stmt_init(conn); + + int32_t ret = taos_stmt_prepare(stmt, sql, 0); + ASSERT_EQ(ret, 0); + + //ts timestamp, k int, a binary(11), b nchar(4) + struct { + int64_t ts; + int k; + char* a; + char* b; + } v = {0}; + + TAOS_BIND params[4]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_INT; + params[1].buffer_length = sizeof(v.k); + params[1].buffer = &v.k; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_BINARY; + params[2].buffer_length = sizeof(v.a); + params[2].buffer = &v.a; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[3].buffer_length = sizeof(v.b); + params[3].buffer = &v.b; + params[3].is_null = NULL; + + v.ts = start_ts + 20; + v.k = 123; + + char str[] = "abc"; + uintptr_t len = strlen(str); + + v.a = str; + params[2].length = &len; + params[2].buffer_length = len; + params[2].buffer = str; + + char nstr[] = "999"; + uintptr_t len1 = strlen(nstr); + + v.b = nstr; + params[3].buffer_length = len1; + params[3].buffer = nstr; + params[3].length = &len1; + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute insert statement.\033[0m\n"); + return; + } + + v.ts = start_ts + 30; + v.k = 911; + + char str1[] = "92"; + len = strlen(str1); + + params[2].length = &len; + params[2].buffer_length = len; + params[2].buffer = str1; + + char nstr1[] = "1920"; + len1 = strlen(nstr1); + + params[3].buffer_length = len1; + params[3].buffer = nstr1; + params[3].length = &len1; + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + + ret = taos_stmt_execute(stmt); + if (ret != 0) { + printf("%d\n", ret); + printf("\033[31mfailed to execute insert statement.\033[0m\n"); + return; + } + + taos_stmt_close(stmt); + taos_close(conn); } -/* test parse time function */ -TEST(testCase, result_field_test) { - taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); - taos_init(); +void validateResultFields() { TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); if (conn == NULL) { printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); @@ -134,5 +236,31 @@ TEST(testCase, result_field_test) { ASSERT_STREQ(fields[6].name, "first(ts)"); taos_free_result(res); + + // update the configure parameter, the result field name will be changed + tsKeepOriginalColumnName = 1; + res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1"); + ASSERT_EQ(taos_num_fields(res), 7); + + fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[0].name, "ts"); + + ASSERT_EQ(fields[2].bytes, 4); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[2].name, "k"); + + taos_free_result(res); + taos_close(conn); } +} +/* test parse time function */ +TEST(testCase, result_field_test) { + taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + taos_init(); + + validateResultFields(); + stmtInsertTest(); +} diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index df0ac7986599da3e53d97f64c0c5113a357f9177..0da7bda994db83882e36e9d52a7983635ad85330 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/query/inc/qArithmeticOperator.h b/src/common/inc/tarithoperator.h similarity index 100% rename from src/query/inc/qArithmeticOperator.h rename to src/common/inc/tarithoperator.h diff --git a/src/common/inc/tcmdtype.h b/src/common/inc/tcmdtype.h index bec85905369b5a1ea4274a9d799c664ce7aee544..be16e80124358012a079ada5f7cd689afa0f7b75 100644 --- a/src/common/inc/tcmdtype.h +++ b/src/common/inc/tcmdtype.h @@ -51,6 +51,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_ACCT, "alter-acct" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" ) + TSDB_DEFINE_SQL_TYPE(TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" ) @@ -87,13 +88,13 @@ enum { */ TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_EMPTY_RESULT, "retrieve-empty-result" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RESET_CACHE, "reset-cache" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_STATUS, "serv-status" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_DB, "current-db" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RESET_CACHE, "reset-cache" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_STATUS, "serv-status" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_DB, "current-db" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SERV_VERSION, "serv-version" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CLI_VERSION, "cli-version" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CLI_VERSION, "cli-version" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CURRENT_USER, "current-user ") - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_LOCAL, "cfg-local" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_LOCAL, "cfg-local" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MAX, "max" ) }; diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 8d4949d9b4364fd1e8c70cbb883aa56468724108..e8c076099759ffe683c78b55a54848cf765daada 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -20,30 +20,30 @@ #include #include "talgo.h" -#include "taosdef.h" +#include "ttype.h" #include "tutil.h" #ifdef __cplusplus extern "C" { #endif -#define STR_TO_VARSTR(x, str) \ - do { \ - VarDataLenT __len = strlen(str); \ - *(VarDataLenT *)(x) = __len; \ - memcpy(varDataVal(x), (str), __len); \ +#define STR_TO_VARSTR(x, str) \ + do { \ + VarDataLenT __len = (VarDataLenT)strlen(str); \ + *(VarDataLenT *)(x) = __len; \ + memcpy(varDataVal(x), (str), __len); \ } while (0); -#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ - do { \ +#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ + do { \ char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \ - varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ + varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ } while (0) -#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ - do { \ - *(VarDataLenT *)(x) = (_size); \ - memcpy(varDataVal(x), (str), (_size)); \ +#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ + do { \ + *(VarDataLenT *)(x) = (VarDataLenT)(_size); \ + memcpy(varDataVal(x), (str), (_size)); \ } while (0); // ----------------- TSDB COLUMN DEFINITION @@ -68,9 +68,9 @@ typedef struct { typedef struct { int version; // version int numOfCols; // Number of columns appended - int tlen; // maximum length of a SDataRow without the header part + int tlen; // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes)) uint16_t flen; // First part length in a SDataRow after the header part - uint16_t vlen; // pure value part length, excluded the overhead + uint16_t vlen; // pure value part length, excluded the overhead (bytes only) STColumn columns[]; } STSchema; @@ -134,6 +134,22 @@ typedef uint64_t TKEY; #define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) #define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) +#define MIN_TS_KEY ((TSKEY)0x8000000000000001) +#define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff) + +#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key)) + +static FORCE_INLINE TKEY keyToTkey(TSKEY key) { + TSKEY lkey = key; + if (key > MAX_TS_KEY) { + lkey = MAX_TS_KEY; + } else if (key < MIN_TS_KEY) { + lkey = MIN_TS_KEY; + } + + return tdGetTKEY(lkey); +} + static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { TSKEY key1 = tdGetKey(*(TKEY *)tkey1); TSKEY key2 = tdGetKey(*(TKEY *)tkey2); @@ -156,7 +172,7 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { * +----------+----------+---------------------------------+---------------------------------+ * | len | sversion | First part | Second part | * +----------+----------+---------------------------------+---------------------------------+ - * + * * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ typedef void *SDataRow; @@ -267,18 +283,43 @@ typedef struct { #define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) -#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0)) -#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0)) -#define dataColsTKeyLast(pCols) \ - (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) -#define dataColsKeyLast(pCols) \ - (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) +static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { + if (pCols->numOfRows) { + return dataColsTKeyAt(pCols, 0); + } else { + return TKEY_INVALID; + } +} + +static FORCE_INLINE TSKEY dataColsKeyFirst(SDataCols *pCols) { + if (pCols->numOfRows) { + return dataColsKeyAt(pCols, 0); + } else { + return TSDB_DATA_TIMESTAMP_NULL; + } +} + +static FORCE_INLINE TKEY dataColsTKeyLast(SDataCols *pCols) { + if (pCols->numOfRows) { + return dataColsTKeyAt(pCols, pCols->numOfRows - 1); + } else { + return TKEY_INVALID; + } +} + +static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) { + if (pCols->numOfRows) { + return dataColsKeyAt(pCols, pCols->numOfRows - 1); + } else { + return TSDB_DATA_TIMESTAMP_NULL; + } +} SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); -void tdFreeDataCols(SDataCols *pCols); +SDataCols *tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); diff --git a/src/query/inc/qAst.h b/src/common/inc/texpr.h similarity index 83% rename from src/query/inc/qAst.h rename to src/common/inc/texpr.h index 39af7261efc222c8a6bcfc809288c256eccb0970..acfbffc01e400f8b111ee92b7651bb048c112bd2 100644 --- a/src/query/inc/qAst.h +++ b/src/common/inc/texpr.h @@ -31,6 +31,15 @@ extern "C" { struct tExprNode; struct SSchema; +#define QUERY_COND_REL_PREFIX_IN "IN|" +#define QUERY_COND_REL_PREFIX_LIKE "LIKE|" + +#define QUERY_COND_REL_PREFIX_IN_LEN 3 +#define QUERY_COND_REL_PREFIX_LIKE_LEN 5 + +typedef bool (*__result_filter_fn_t)(const void *, void *); +typedef void (*__do_filter_suppl_fn_t)(void *, void *); + enum { TSQL_NODE_DUMMY = 0x0, TSQL_NODE_EXPR = 0x1, @@ -38,9 +47,6 @@ enum { TSQL_NODE_VALUE = 0x4, }; -typedef bool (*__result_filter_fn_t)(const void *, void *); -typedef void (*__do_filter_suppl_fn_t)(void *, void *); - /** * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required */ @@ -52,12 +58,6 @@ typedef struct tQueryInfo { bool indexed; // indexed columns } tQueryInfo; -typedef struct SExprTraverseSupp { - __result_filter_fn_t nodeFilterFn; - __do_filter_suppl_fn_t setupInfoFn; - void * pExtInfo; -} SExprTraverseSupp; - typedef struct tExprNode { uint8_t nodeType; union { @@ -65,7 +65,7 @@ typedef struct tExprNode { uint8_t optr; // filter operator uint8_t hasPK; // 0: do not contain primary filter, 1: contain void * info; // support filter operation on this expression only available for leaf node - + struct tExprNode *pLeft; // left child pointer struct tExprNode *pRight; // right child pointer } _node; @@ -74,19 +74,27 @@ typedef struct tExprNode { }; } tExprNode; -void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, - char *(*cb)(void *, const char*, int32_t)); +typedef struct SExprTraverseSupp { + __result_filter_fn_t nodeFilterFn; + __do_filter_suppl_fn_t setupInfoFn; + void * pExtInfo; +} SExprTraverseSupp; + +void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); tExprNode* exprTreeFromBinary(const void* data, size_t size); tExprNode* exprTreeFromTableName(const char* tbnameCond); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); -void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); -void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*)); - bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param); +typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, + int32_t rightType, void *output, int32_t order); + +void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, + char *(*cb)(void *, const char*, int32_t)); + #ifdef __cplusplus } #endif diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index c54d519637cad547fac2d1960e809ec2c615274d..3f96466cc00fc3250860e79bc3dd85e6fd0368d7 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_COMMON_GLOBAL_H #define TDENGINE_COMMON_GLOBAL_H +#include "taosdef.h" + #ifdef __cplusplus extern "C" { #endif @@ -57,7 +59,8 @@ extern int32_t tsCompressMsgSize; extern char tsTempDir[]; //query buffer management -extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing +extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing +extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node during query processing extern int32_t tsRetrieveBlockingModel;// retrieve threads will be blocked extern int8_t tsKeepOriginalColumnName; @@ -88,10 +91,11 @@ extern int32_t tsMinRowsInFileBlock; extern int32_t tsMaxRowsInFileBlock; extern int16_t tsCommitTime; // seconds extern int32_t tsTimePrecision; -extern int16_t tsCompression; -extern int16_t tsWAL; +extern int8_t tsCompression; +extern int8_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; +extern int16_t tsPartitons; extern int32_t tsQuorum; extern int8_t tsUpdate; extern int8_t tsCacheLastRow; @@ -146,7 +150,6 @@ extern char tsDataDir[]; extern char tsLogDir[]; extern char tsScriptDir[]; extern int64_t tsMsPerDay[3]; -extern char tsVnodeBakDir[]; // system info extern char tsOsName[]; @@ -160,6 +163,7 @@ extern float tsTotalDataDirGB; extern float tsAvailLogDirGB; extern float tsAvailTmpDirectorySpace; extern float tsAvailDataDirGB; +extern float tsUsedDataDirGB; extern float tsMinimalLogDirGB; extern float tsReservedTmpDirectorySpace; extern float tsMinimalDataDirGB; @@ -180,7 +184,7 @@ extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; extern int32_t vDebugFlag; extern int32_t mDebugFlag; -extern int32_t cDebugFlag; +extern uint32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; extern int32_t sdbDebugFlag; @@ -195,6 +199,14 @@ extern int32_t wDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; +typedef struct { + char dir[TSDB_FILENAME_LEN]; + int level; + int primary; +} SDiskCfg; +extern int32_t tsDiskCfgNum; +extern SDiskCfg tsDiskCfg[]; + #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) void taosInitGlobalCfg(); @@ -203,6 +215,9 @@ void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); +void taosAddDataDir(int index, char *v1, int level, int primary); +void taosReadDataDirCfg(char *v1, char *v2, char *v3); +void taosPrintDataDirCfg(); #ifdef __cplusplus } diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 9e0093ebfed67a2cb4ca924d053ccffd38e316e6..465b298973cbccdf380150bdce46a09b56594801 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifndef TDENGINE_NAME_H #define TDENGINE_NAME_H @@ -18,9 +33,28 @@ typedef struct SDataStatis { typedef struct SColumnInfoData { SColumnInfo info; - void* pData; // the corresponding block data in memory + char* pData; // the corresponding block data in memory } SColumnInfoData; +typedef struct SResPair { + TSKEY key; + double avg; +} SResPair; + +#define TSDB_DB_NAME_T 1 +#define TSDB_TABLE_NAME_T 2 + +#define T_NAME_ACCT 0x1u +#define T_NAME_DB 0x2u +#define T_NAME_TABLE 0x4u + +typedef struct SName { + uint8_t type; //db_name_t, table_name_t + char acctId[TSDB_ACCT_ID_LEN]; + char dbname[TSDB_DB_NAME_LEN]; + char tname[TSDB_TABLE_NAME_LEN]; +} SName; + void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); @@ -29,14 +63,52 @@ size_t tableIdPrefix(const char* name, char* prefix, int32_t len); void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); -SSchema tGetTableNameColumnSchema(); +//SSchema tGetTbnameColumnSchema(); + +SSchema tGetBlockDistColumnSchema(); SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name); bool tscValidateTableNameLength(size_t len); -SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); +SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters); + +SSchema* tGetTbnameColumnSchema(); + +/** + * check if the schema is valid or not, including following aspects: + * 1. number of columns + * 2. column types + * 3. column length + * 4. column names + * 5. total length + * + * @param pSchema + * @param numOfCols + * @return + */ +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); + +int32_t tNameExtractFullName(const SName* name, char* dst); +int32_t tNameLen(const SName* name); + +SName* tNameDup(const SName* name); + +bool tIsValidName(const SName* name); + +const char* tNameGetTableName(const SName* name); + +int32_t tNameGetDbName(const SName* name, char* dst); +int32_t tNameGetFullDbName(const SName* name, char* dst); + +bool tNameIsEmpty(const SName* name); + +void tNameAssign(SName* dst, const SName* src); + +int32_t tNameFromString(SName* dst, const char* str, uint32_t type); + +int32_t tNameSetAcctId(SName* dst, const char* acct); -SSchema tscGetTbnameColumnSchema(); +int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken); #endif // TDENGINE_NAME_H diff --git a/src/query/src/qArithmeticOperator.c b/src/common/src/tarithoperator.c similarity index 63% rename from src/query/src/qArithmeticOperator.c rename to src/common/src/tarithoperator.c index 27bdd4372b98f94e232a1d43a514f98adb28a598..1cb667d259f040cfab0656562f7c97444fc48d8a 100644 --- a/src/query/src/qArithmeticOperator.c +++ b/src/common/src/tarithoperator.c @@ -15,9 +15,9 @@ #include "os.h" -#include "qArithmeticOperator.h" -#include "taosdef.h" +#include "ttype.h" #include "tutil.h" +#include "tarithoperator.h" #define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \ { \ @@ -145,7 +145,7 @@ void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + SET_DOUBLE_NULL(pOutput); continue; } @@ -154,7 +154,7 @@ void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight } else if (numLeft == 1) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + SET_DOUBLE_NULL(pOutput); continue; } @@ -163,7 +163,7 @@ void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight } else if (numRight == 1) { for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + SET_DOUBLE_NULL(pOutput); continue; } *pOutput = (double)pLeft[i] + pRight[0]; @@ -2556,1123 +2556,6 @@ void vectorRemainder(void *left, int32_t numLeft, int32_t leftType, void *right, } } -/* -void calc_i32_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t); - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i32_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i8_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_u8_u8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, uint8_t, uint8_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_UTINYINT, TSDB_DATA_TYPE_UTINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_u8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, uint8_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_UTINYINT, numLeft, numRight, pOutput, order); -} - -void calc_u8_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_u8_add(right, left, numRight, numLeft, output, order); -} - -void calc_i8_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_i8_add(right, left, numRight, numLeft, output, order); -} - -void calc_i8_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i8_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i16_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_i16_add(right, left, numRight, numLeft, output, order); -} - -void calc_i16_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_i16_add(right, left, numRight, numLeft, output, order); -} - -void calc_i16_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i16_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i64_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_i64_add(right, left, numRight, numLeft, output, order); -} - -void calc_i64_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i16_i64_add(right, left, numRight, numLeft, output, order); -} - -void calc_i64_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_i64_add(right, left, numRight, numLeft, output, order); -} - -void calc_i64_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i64_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_f_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_f_add(right, left, numRight, numLeft, output, order); -} - -void calc_f_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i16_f_add(right, left, numRight, numLeft, output, order); -} - -void calc_f_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_f_add(right, left, numRight, numLeft, output, order); -} - -void calc_f_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i64_f_add(right, left, numRight, numLeft, output, order); -} - -void calc_f_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_f_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_d_i8_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_d_i16_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i16_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_d_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_d_i64_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i64_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_d_f_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_f_d_add(right, left, numRight, numLeft, output, order); -} - -void calc_d_d_add(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_ADD(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -///////////////////////////////////////////////////////////////////////////////////////////////////////// -void calc_i32_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] - pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[0] - pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&pLeft[i], TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] - pRight[0]; - } - } -} - -void calc_i32_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i32_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i8_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i8_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i16_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i16_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i64_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i64_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_f_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_f_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_f_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_f_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_f_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_f_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_d_i8_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int8_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_d_i16_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int16_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_d_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int32_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_d_i64_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int64_t) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_d_f_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, float) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_d_d_sub(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_SUB(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -///////////////////////////////////////////////////////////////////////////////////////////////////////// -void calc_i32_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] * pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] * pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] * pRight[0]; - } - } -} - -void calc_i32_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i32_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i8_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_i8_multi(right, left, numRight, numLeft, output, order); -} - -void calc_i8_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i8_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i16_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_i16_multi(right, left, numRight, numLeft, output, order); -} - -void calc_i16_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_i16_multi(right, left, numRight, numLeft, output, order); -} - -void calc_i16_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i16_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i64_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_i64_multi(right, left, numRight, numLeft, output, order); -} - -void calc_i64_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i16_i64_multi(right, left, numRight, numLeft, output, order); -} - -void calc_i64_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_i64_multi(right, left, numRight, numLeft, output, order); -} - -void calc_i64_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i64_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_f_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_f_multi(right, left, numRight, numLeft, output, order); -} - -void calc_f_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i16_f_multi(right, left, numRight, numLeft, output, order); -} - -void calc_f_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_f_multi(right, left, numRight, numLeft, output, order); -} - -void calc_f_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i64_f_multi(right, left, numRight, numLeft, output, order); -} - -void calc_f_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_f_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_d_i8_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i8_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_d_i16_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i16_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_d_i32_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i32_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_d_i64_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_i64_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_d_f_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - calc_f_d_multi(right, left, numRight, numLeft, output, order); -} - -void calc_d_d_multi(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_MULTI(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////// -void calc_i32_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] / pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] / pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - *pOutput = (double)pLeft[i] / pRight[0]; - } - } -} - -void calc_i32_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i32_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i8_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i8_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i16_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i16_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i64_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i64_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_f_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_f_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_f_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_f_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_f_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_f_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_d_i8_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int8_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_d_i16_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int16_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_d_i32_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int32_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_d_i64_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int64_t) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_d_f_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, float) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_d_d_div(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_DIV(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////////// - -void calc_i32_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - int32_t *pRight = (int32_t *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[i])) * pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] - ((int64_t)(((double)pLeft[0]) / pRight[i])) * pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[0])) * pRight[0]; - } - } -} - -void calc_i32_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i32_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int32_t, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i32_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - int32_t *pLeft = (int32_t *)left; - double * pRight = (double *)right; - double * pOutput = (double *)output; - - int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; - int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; - - if (numLeft == numRight) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[i])) * pRight[i]; - } - } else if (numLeft == 1) { - for (; i >= 0 && i < numRight; i += step, pOutput += 1) { - if (isNull((char *)(pLeft), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[0] - ((int64_t)(((double)pLeft[0]) / pRight[i])) * pRight[i]; - } - } else if (numRight == 1) { - for (; i >= 0 && i < numLeft; i += step, pOutput += 1) { - if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)(pRight), TSDB_DATA_TYPE_INT)) { - setNull((char *)pOutput, TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); - continue; - } - - *pOutput = (double)pLeft[i] - ((int64_t)(((double)pLeft[i]) / pRight[0])) * pRight[0]; - } - } -} - -void calc_i8_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i8_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i8_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i8_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int8_t, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_TINYINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i16_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i16_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i16_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i16_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int16_t, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_i64_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_i64_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_i64_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_i64_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, int64_t, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_BIGINT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_f_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_f_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_f_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_f_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_f_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_f_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, float, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_FLOAT, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -void calc_d_i8_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int8_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_TINYINT, numLeft, numRight, pOutput, order); -} - -void calc_d_i16_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int16_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_SMALLINT, numLeft, numRight, pOutput, order); -} - -void calc_d_i32_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int32_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_INT, numLeft, numRight, pOutput, order); -} - -void calc_d_i64_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, int64_t) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_BIGINT, numLeft, numRight, pOutput, order); -} - -void calc_d_f_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, float) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_FLOAT, numLeft, numRight, pOutput, order); -} - -void calc_d_d_rem(void *left, void *right, int32_t numLeft, int32_t numRight, void *output, int32_t order) { - TYPE_CONVERT(left, right, output, double, double) - ARRAY_LIST_REM(pLeft, pRight, TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, numLeft, numRight, pOutput, order); -} - -* - * the following are two-dimensional array list of callback function . - */ -//_arithmetic_operator_fn_t add_function_arraylist[15][15] = { -// /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ -// {0}, // EMPTY, -// {0}, // TSDB_DATA_TYPE_BOOL, -// {NULL, NULL, calc_i8_i8_add, calc_i8_i16_add, calc_i8_i32_add, calc_i8_i64_add, calc_i8_f_add, calc_i8_d_add, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT -// {NULL, NULL, calc_i16_i8_add, calc_i16_i16_add, calc_i16_i32_add, calc_i16_i64_add, calc_i16_f_add, calc_i16_d_add, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT -// {NULL, NULL, calc_i32_i8_add, calc_i32_i16_add, calc_i32_i32_add, calc_i32_i64_add, calc_i32_f_add, calc_i32_d_add, NULL, NULL}, // TSDB_DATA_TYPE_INT -// {NULL, NULL, calc_i64_i8_add, calc_i64_i16_add, calc_i64_i32_add, calc_i64_i64_add, calc_i64_f_add, calc_i64_d_add, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT -// {NULL, NULL, calc_f_i8_add, calc_f_i16_add, calc_f_i32_add, calc_f_i64_add, calc_f_f_add, calc_f_d_add, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT -// {NULL, NULL, calc_d_i8_add, calc_d_i16_add, calc_d_i32_add, calc_d_i64_add, calc_d_f_add, calc_d_d_add, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -// {0}, // TSDB_DATA_TYPE_BINARY, -// {0}, // TSDB_DATA_TYPE_NCHAR, -// {NULL, NULL, calc_u8_i8_add, calc_u8_i16_add, calc_u8_i32_add, calc_u8_i64_add, calc_u8_f_add, calc_u8_d_add, NULL, NULL, calc_u8_u8_add, calc_u8_u16_add, calc_u8_u32_add, calc_u8_u64_add, NULL}, // TSDB_DATA_TYPE_UTINYINT, -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_USMALLINT, -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_UINT, -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_UBIGINT, -// -//}; -// -//_arithmetic_operator_fn_t sub_function_arraylist[8][15] = { -// /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ -// {0}, // EMPTY, -// {0}, // TSDB_DATA_TYPE_BOOL, -// {NULL, NULL, calc_i8_i8_sub, calc_i8_i16_sub, calc_i8_i32_sub, calc_i8_i64_sub, calc_i8_f_sub, calc_i8_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT -// {NULL, NULL, calc_i16_i8_sub, calc_i16_i16_sub, calc_i16_i32_sub, calc_i16_i64_sub, calc_i16_f_sub, calc_i16_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT -// {NULL, NULL, calc_i32_i8_sub, calc_i32_i16_sub, calc_i32_i32_sub, calc_i32_i64_sub, calc_i32_f_sub, calc_i32_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_INT -// {NULL, NULL, calc_i64_i8_sub, calc_i64_i16_sub, calc_i64_i32_sub, calc_i64_i64_sub, calc_i64_f_sub, calc_i64_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT -// {NULL, NULL, calc_f_i8_sub, calc_f_i16_sub, calc_f_i32_sub, calc_f_i64_sub, calc_f_f_sub, calc_f_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT -// {NULL, NULL, calc_d_i8_sub, calc_d_i16_sub, calc_d_i32_sub, calc_d_i64_sub, calc_d_f_sub, calc_d_d_sub, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -//}; -// -//_arithmetic_operator_fn_t multi_function_arraylist[][15] = { -// /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, -// {NULL, NULL, calc_i8_i8_multi, calc_i8_i16_multi, calc_i8_i32_multi, calc_i8_i64_multi, calc_i8_f_multi, calc_i8_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT -// {NULL, NULL, calc_i16_i8_multi, calc_i16_i16_multi, calc_i16_i32_multi, calc_i16_i64_multi, calc_i16_f_multi, calc_i16_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT -// {NULL, NULL, calc_i32_i8_multi, calc_i32_i16_multi, calc_i32_i32_multi, calc_i32_i64_multi, calc_i32_f_multi, calc_i32_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_INT -// {NULL, NULL, calc_i64_i8_multi, calc_i64_i16_multi, calc_i64_i32_multi, calc_i64_i64_multi, calc_i64_f_multi, calc_i64_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT -// {NULL, NULL, calc_f_i8_multi, calc_f_i16_multi, calc_f_i32_multi, calc_f_i64_multi, calc_f_f_multi, calc_f_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT -// {NULL, NULL, calc_d_i8_multi, calc_d_i16_multi, calc_d_i32_multi, calc_d_i64_multi, calc_d_f_multi, calc_d_d_multi, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -//}; -// -//_arithmetic_operator_fn_t div_function_arraylist[8][15] = { -// /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary*/ -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, -// {NULL, NULL, calc_i8_i8_div, calc_i8_i16_div, calc_i8_i32_div, calc_i8_i64_div, calc_i8_f_div, calc_i8_d_div, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT -// {NULL, NULL, calc_i16_i8_div, calc_i16_i16_div, calc_i16_i32_div, calc_i16_i64_div, calc_i16_f_div, calc_i16_d_div, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT -// {NULL, NULL, calc_i32_i8_div, calc_i32_i16_div, calc_i32_i32_div, calc_i32_i64_div, calc_i32_f_div, calc_i32_d_div, NULL, NULL}, // TSDB_DATA_TYPE_INT -// {NULL, NULL, calc_i64_i8_div, calc_i64_i16_div, calc_i64_i32_div, calc_i64_i64_div, calc_i64_f_div, calc_i64_d_div, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT -// {NULL, NULL, calc_f_i8_div, calc_f_i16_div, calc_f_i32_div, calc_f_i64_div, calc_f_f_div, calc_f_d_div, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT -// {NULL, NULL, calc_d_i8_div, calc_d_i16_div, calc_d_i32_div, calc_d_i64_div, calc_d_f_div, calc_d_d_div, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -//}; -// -//_arithmetic_operator_fn_t rem_function_arraylist[8][15] = { -// /*NULL, bool, tinyint, smallint, int, bigint, float, double, timestamp, binary, nchar, unsigned tinyint, unsigned smallint, unsigned int, unsigned bigint*/ -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // EMPTY, -// {NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL}, // TSDB_DATA_TYPE_BOOL, -// {NULL, NULL, calc_i8_i8_rem, calc_i8_i16_rem, calc_i8_i32_rem, calc_i8_i64_rem, calc_i8_f_rem, calc_i8_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_TINYINT -// {NULL, NULL, calc_i16_i8_rem, calc_i16_i16_rem, calc_i16_i32_rem, calc_i16_i64_rem, calc_i16_f_rem, calc_i16_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_SMALLINT -// {NULL, NULL, calc_i32_i8_rem, calc_i32_i16_rem, calc_i32_i32_rem, calc_i32_i64_rem, calc_i32_f_rem, calc_i32_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_INT -// {NULL, NULL, calc_i64_i8_rem, calc_i64_i16_rem, calc_i64_i32_rem, calc_i64_i64_rem, calc_i64_f_rem, calc_i64_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_BIGINT -// {NULL, NULL, calc_f_i8_rem, calc_f_i16_rem, calc_f_i32_rem, calc_f_i64_rem, calc_f_f_rem, calc_f_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_FLOAT -// {NULL, NULL, calc_d_i8_rem, calc_d_i16_rem, calc_d_i32_rem, calc_d_i64_rem, calc_d_f_rem, calc_d_d_rem, NULL, NULL}, // TSDB_DATA_TYPE_DOUBLE -//}; - -//////////////////////////////////////////////////////////////////////////////////////////////////////////// - _arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr) { switch (arithmeticOptr) { case TSDB_BINARY_OP_ADD: diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index f21205479396ba606a1212f30350df2e0b3f59b5..f5b84e4c9ad7492ecea11eb232b33f373d37235a 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -289,23 +289,31 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { return NULL; } - pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); - if (pCols->cols == NULL) { - uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); - tdFreeDataCols(pCols); - return NULL; + pCols->maxPoints = maxRows; + + if (maxCols > 0) { + pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); + if (pCols->cols == NULL) { + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, + strerror(errno)); + tdFreeDataCols(pCols); + return NULL; + } + + pCols->maxCols = maxCols; } pCols->maxRowSize = maxRowSize; - pCols->maxCols = maxCols; - pCols->maxPoints = maxRows; pCols->bufSize = maxRowSize * maxRows; - pCols->buf = malloc(pCols->bufSize); - if (pCols->buf == NULL) { - uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno)); - tdFreeDataCols(pCols); - return NULL; + if (pCols->bufSize > 0) { + pCols->buf = malloc(pCols->bufSize); + if (pCols->buf == NULL) { + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, + strerror(errno)); + tdFreeDataCols(pCols); + return NULL; + } } return pCols; @@ -337,12 +345,13 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { return 0; } -void tdFreeDataCols(SDataCols *pCols) { +SDataCols *tdFreeDataCols(SDataCols *pCols) { if (pCols) { tfree(pCols->buf); tfree(pCols->cols); free(pCols); } + return NULL; } SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { diff --git a/src/query/src/qAst.c b/src/common/src/texpr.c similarity index 94% rename from src/query/src/qAst.c rename to src/common/src/texpr.c index 1e6dbe8e3dd52c98896c02954b2208c1cfcf7a50..1008c4cf8f77ca77f59a57aea189cdebef9c9129 100644 --- a/src/query/src/qAst.c +++ b/src/common/src/texpr.c @@ -16,18 +16,15 @@ #include "os.h" #include "exception.h" -#include "qArithmeticOperator.h" -#include "qAst.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" -#include "tname.h" -#include "tschemautil.h" #include "tsdb.h" #include "tskiplist.h" -#include "tsqlfunction.h" +#include "texpr.h" +#include "tarithoperator.h" static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) { if (pLeft->nodeType == TSQL_NODE_COL) { @@ -44,41 +41,46 @@ static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, co static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { switch(type) { - case TSDB_DATA_TYPE_TINYINT: { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT:{ int8_t* p = (int8_t*) dest; int8_t* pSrc = (int8_t*) src; for(int32_t i = 0; i < numOfRows; ++i) { p[i] = pSrc[numOfRows - i - 1]; } - break; + return; } - case TSDB_DATA_TYPE_SMALLINT: { + + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT:{ int16_t* p = (int16_t*) dest; int16_t* pSrc = (int16_t*) src; for(int32_t i = 0; i < numOfRows; ++i) { p[i] = pSrc[numOfRows - i - 1]; } - break; + return; } - case TSDB_DATA_TYPE_INT: { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { int32_t* p = (int32_t*) dest; int32_t* pSrc = (int32_t*) src; for(int32_t i = 0; i < numOfRows; ++i) { p[i] = pSrc[numOfRows - i - 1]; } - break; + return; } - case TSDB_DATA_TYPE_BIGINT: { + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: { int64_t* p = (int64_t*) dest; int64_t* pSrc = (int64_t*) src; for(int32_t i = 0; i < numOfRows; ++i) { p[i] = pSrc[numOfRows - i - 1]; } - break; + return; } case TSDB_DATA_TYPE_FLOAT: { float* p = (float*) dest; @@ -87,7 +89,7 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf for(int32_t i = 0; i < numOfRows; ++i) { p[i] = pSrc[numOfRows - i - 1]; } - break; + return; } case TSDB_DATA_TYPE_DOUBLE: { double* p = (double*) dest; @@ -96,19 +98,21 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf for(int32_t i = 0; i < numOfRows; ++i) { p[i] = pSrc[numOfRows - i - 1]; } - break; + return; } default: assert(0); } } -void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { +static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)); + +void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { if (pNode == NULL) { return; } if (pNode->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&pNode, fp); + doExprTreeDestroy(&pNode, fp); } else if (pNode->nodeType == TSQL_NODE_VALUE) { tVariantDestroy(pNode->pVal); } else if (pNode->nodeType == TSQL_NODE_COL) { @@ -118,14 +122,14 @@ void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) { free(pNode); } -void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { +static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { if (*pExpr == NULL) { return; } if ((*pExpr)->nodeType == TSQL_NODE_EXPR) { - tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); - tExprTreeDestroy(&(*pExpr)->_node.pRight, fp); + doExprTreeDestroy(&(*pExpr)->_node.pLeft, fp); + doExprTreeDestroy(&(*pExpr)->_node.pRight, fp); if (fp != NULL) { fp((*pExpr)->_node.info); @@ -270,8 +274,9 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, } } - free(pLeftOutput); - free(pRightOutput); + tfree(pdata); + tfree(pLeftOutput); + tfree(pRightOutput); } static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) { @@ -342,7 +347,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) { } tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode)); - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, pExpr, NULL); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, pExpr, NULL); pExpr->nodeType = tbufReadUint8(br); if (pExpr->nodeType == TSQL_NODE_VALUE) { @@ -396,7 +401,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { int32_t anchor = CLEANUP_GET_ANCHOR(); tExprNode* expr = exception_calloc(1, sizeof(tExprNode)); - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL); expr->nodeType = TSQL_NODE_EXPR; @@ -407,7 +412,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) { SSchema* pSchema = exception_calloc(1, sizeof(SSchema)); left->pSchema = pSchema; - *pSchema = tscGetTbnameColumnSchema(); + *pSchema = *tGetTbnameColumnSchema(); tExprNode* right = exception_calloc(1, sizeof(tExprNode)); expr->_node.pRight = right; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 125e80564295401cb17337f09e66cf4955ecd944..5f4ce046ed5359bab8c0f175ff0c08d6da939700 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -47,11 +47,11 @@ char tsEmail[TSDB_FQDN_LEN] = {0}; // common int32_t tsRpcTimer = 1000; int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsMaxShellConns = 5000; +int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; -int32_t tsNumOfCommitThreads = 1; +int32_t tsNumOfCommitThreads = 4; float tsRatioOfQueryCores = 1.0f; int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; @@ -59,7 +59,6 @@ char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; -char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -72,7 +71,7 @@ char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; int32_t tsCompressMsgSize = -1; // client -int32_t tsMaxSQLStringLen = TSDB_MAX_SQL_LEN; +int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; int8_t tsTscEnableRecordSql = 0; // the maximum number of results for projection query on super table that are returned from @@ -105,6 +104,7 @@ int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance // 0 no query allowed, queries are disabled // positive value (in MB) int32_t tsQueryBufferSize = -1; +int64_t tsQueryBufferSizeBytes = -1; // in retrieve blocking model, the retrieve threads will wait for the completion of the query processing. int32_t tsRetrieveBlockingModel = 0; @@ -121,13 +121,14 @@ int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION; -int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; -int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; +int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL; +int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; +int16_t tsPartitons = TSDB_DEFAULT_DB_PARTITON_OPTION; int8_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; -int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_BLOCK_SIZE; +int8_t tsCacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; @@ -137,7 +138,7 @@ int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; int8_t tsEnableBalance = 1; int8_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds -int32_t tsOfflineThreshold = 86400 * 100; // seconds 10days +int32_t tsOfflineThreshold = 86400 * 100; // seconds 100 days int32_t tsMnodeEqualVnodeNum = 4; int8_t tsEnableFlowCtrl = 1; int8_t tsEnableSlaveQuery = 1; @@ -181,7 +182,15 @@ char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDataDir[TSDB_FILENAME_LEN] = {0}; char tsScriptDir[TSDB_FILENAME_LEN] = {0}; -char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; +char tsTempDir[TSDB_FILENAME_LEN] = "/tmp/"; + +int32_t tsDiskCfgNum = 0; + +#ifndef _STORAGE +SDiskCfg tsDiskCfg[1]; +#else +SDiskCfg tsDiskCfg[TSDB_MAX_DISKS]; +#endif /* * minimum scale for whole system, millisecond by default @@ -201,6 +210,7 @@ float tsTotalTmpDirGB = 0; float tsTotalDataDirGB = 0; float tsAvailTmpDirectorySpace = 0; float tsAvailDataDirGB = 0; +float tsUsedDataDirGB = 0; float tsReservedTmpDirectorySpace = 1.0f; float tsMinimalDataDirGB = 1.0f; int32_t tsTotalMemoryMB = 0; @@ -212,7 +222,7 @@ int32_t mDebugFlag = 131; int32_t sdbDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; -int32_t cDebugFlag = 131; +uint32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; int32_t httpDebugFlag = 131; @@ -226,6 +236,7 @@ int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t cqDebugFlag = 131; +int32_t fsDebugFlag = 135; int32_t (*monStartSystemFp)() = NULL; void (*monStopSystemFp)() = NULL; @@ -264,7 +275,7 @@ bool taosCfgDynamicOptions(char *msg) { int32_t vint = 0; paGetToken(msg, &option, &olen); - if (olen == 0) return TSDB_CODE_COM_INVALID_CFG_MSG; + if (olen == 0) return false;; paGetToken(option + olen + 1, &value, &vlen); if (vlen == 0) @@ -283,7 +294,7 @@ bool taosCfgDynamicOptions(char *msg) { int32_t cfgLen = (int32_t)strlen(cfg->option); if (cfgLen != olen) continue; if (strncasecmp(option, cfg->option, olen) != 0) continue; - if (cfg->valType != TAOS_CFG_VTYPE_INT32) { + if (cfg->valType == TAOS_CFG_VTYPE_INT32) { *((int32_t *)cfg->ptr) = vint; } else { *((int8_t *)cfg->ptr) = (int8_t)vint; @@ -307,11 +318,9 @@ bool taosCfgDynamicOptions(char *msg) { } return true; } - if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { - taosSetAllDebugFlag(); + taosSetAllDebugFlag(); } - return true; } @@ -333,6 +342,56 @@ bool taosCfgDynamicOptions(char *msg) { return false; } +void taosAddDataDir(int index, char *v1, int level, int primary) { + tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); + tsDiskCfg[index].level = level; + tsDiskCfg[index].primary = primary; + uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); +} + +#ifndef _STORAGE +void taosReadDataDirCfg(char *v1, char *v2, char *v3) { + if (tsDiskCfgNum == 1) { + SDiskCfg *cfg = &tsDiskCfg[0]; + uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); + } + taosAddDataDir(0, v1, 0, 1); + tsDiskCfgNum = 1; +} + +void taosPrintDataDirCfg() { + for (int i = 0; i < tsDiskCfgNum; ++i) { + SDiskCfg *cfg = &tsDiskCfg[i]; + uInfo(" dataDir: %s", cfg->dir); + } +} +#endif + +static void taosCheckDataDirCfg() { + if (tsDiskCfgNum <= 0) { + taosAddDataDir(0, tsDataDir, 0, 1); + tsDiskCfgNum = 1; + uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); + } +} + +static int32_t taosCheckTmpDir(void) { + if (strlen(tsTempDir) <= 0){ + uError("tempDir is not set"); + return -1; + } + + DIR *dir = opendir(tsTempDir); + if (dir == NULL) { + uError("can not open tempDir:%s, error:%s", tsTempDir, strerror(errno)); + return -1; + } + + closedir(dir); + + return 0; +} + static void doInitGlobalConfig(void) { osInit(); srand(taosSafeRand()); @@ -373,10 +432,10 @@ static void doInitGlobalConfig(void) { // port cfg.option = "serverPort"; cfg.ptr = &tsServerPort; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_UINT16; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; cfg.minValue = 1; - cfg.maxValue = 65535; + cfg.maxValue = 65056; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); @@ -414,7 +473,7 @@ static void doInitGlobalConfig(void) { cfg.option = "dataDir"; cfg.ptr = tsDataDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; + cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; cfg.minValue = 0; cfg.maxValue = 0; @@ -550,7 +609,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = 3; - cfg.maxValue = 7200000; + cfg.maxValue = 86400 * 365; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_SECOND; taosInitConfigOption(cfg); @@ -758,7 +817,7 @@ static void doInitGlobalConfig(void) { cfg.option = "comp"; cfg.ptr = &tsCompression; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_COMP_LEVEL; cfg.maxValue = TSDB_MAX_COMP_LEVEL; @@ -768,7 +827,7 @@ static void doInitGlobalConfig(void) { cfg.option = "walLevel"; cfg.ptr = &tsWAL; - cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.valType = TAOS_CFG_VTYPE_INT8; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = TSDB_MIN_WAL_LEVEL; cfg.maxValue = TSDB_MAX_WAL_LEVEL; @@ -796,6 +855,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "partitions"; + cfg.ptr = &tsPartitons; + cfg.valType = TAOS_CFG_VTYPE_INT16; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_DB_PARTITON_OPTION; + cfg.maxValue = TSDB_MAX_DB_PARTITON_OPTION; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "quorum"; cfg.ptr = &tsQuorum; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1094,7 +1163,7 @@ static void doInitGlobalConfig(void) { cfg.ptr = &tsHttpMaxThreads; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; + cfg.minValue = 2; cfg.maxValue = 1000000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; @@ -1447,6 +1516,12 @@ int32_t taosCheckGlobalCfg() { snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port); } + taosCheckDataDirCfg(); + + if (taosCheckTmpDir()) { + return -1; + } + taosGetSystemInfo(); tsSetLocale(); @@ -1460,6 +1535,13 @@ int32_t taosCheckGlobalCfg() { tsNumOfCores = 1; } + if (tsHttpMaxThreads == 2) { + int32_t halfNumOfCores = tsNumOfCores >> 1; + if (halfNumOfCores > 2) { + tsHttpMaxThreads = halfNumOfCores; + } + } + if (tsMaxTablePerVnode < tsMinTablePerVnode) { uError("maxTablesPerVnode(%d) < minTablesPerVnode(%d), reset to minTablesPerVnode(%d)", tsMaxTablePerVnode, tsMinTablePerVnode, tsMinTablePerVnode); @@ -1488,6 +1570,12 @@ int32_t taosCheckGlobalCfg() { tsSyncPort = tsServerPort + TSDB_PORT_SYNC; tsHttpPort = tsServerPort + TSDB_PORT_HTTP; + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + } + + uInfo(" check global cfg completed"); + uInfo("=================================="); taosPrintGlobalCfg(); return 0; diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 5c351edf48e10ff3176f38a177142f0f394ee5bd..787aa1e95b0b361d1fe4fc6931e7da04aa419c65 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -3,28 +3,14 @@ #include "tname.h" #include "tstoken.h" -#include "ttokendef.h" #include "tvariant.h" -// todo refactor -UNUSED_FUNC static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { - for (int32_t i = 0; i < num; ++i) { - while (*input != 0 && *input++ != delim) { - }; - } - return input; -} +#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) +#define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) -UNUSED_FUNC static FORCE_INLINE size_t copy(char* dst, const char* src, char delimiter) { - size_t len = 0; - while (*src != delimiter && *src != 0) { - *dst++ = *src++; - len++; - } - - return len; -} +#define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) +//TODO remove it void extractTableName(const char* tableId, char* name) { size_t s1 = strcspn(tableId, &TS_PATH_DELIMITER[0]); size_t s2 = strcspn(&tableId[s1 + 1], &TS_PATH_DELIMITER[0]); @@ -39,6 +25,7 @@ char* extractDBName(const char* tableId, char* name) { return strncpy(name, &tableId[offset1 + 1], len); } +// todo remove it size_t tableIdPrefix(const char* name, char* prefix, int32_t len) { tstrncpy(prefix, name, len); strcat(prefix, TS_PATH_DELIMITER); @@ -46,12 +33,12 @@ size_t tableIdPrefix(const char* name, char* prefix, int32_t len) { return strlen(prefix); } -SSchema tGetTableNameColumnSchema() { +SSchema tGetBlockDistColumnSchema() { SSchema s = {0}; - s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; + s.bytes = TSDB_MAX_BINARY_LEN;; s.type = TSDB_DATA_TYPE_BINARY; - s.colId = TSDB_TBNAME_COLUMN_INDEX; - tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + s.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; + tstrncpy(s.name, TSQL_BLOCK_DIST_L, TSDB_COL_NAME_LEN); return s; } @@ -62,7 +49,7 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); } else { - s.bytes = tDataTypeDesc[pVal->nType].nSize; + s.bytes = tDataTypes[pVal->nType].bytes; } s.colId = TSDB_UD_COLUMN_INDEX; @@ -81,7 +68,7 @@ bool tscValidateTableNameLength(size_t len) { return len < TSDB_TABLE_NAME_LEN; } -SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) { +SColumnFilterInfo* tFilterInfoDup(const SColumnFilterInfo* src, int32_t numOfFilters) { if (numOfFilters == 0) { assert(src == NULL); return NULL; @@ -196,13 +183,256 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) { } } -SSchema tscGetTbnameColumnSchema() { - struct SSchema s = { - .colId = TSDB_TBNAME_COLUMN_INDEX, - .type = TSDB_DATA_TYPE_BINARY, - .bytes = TSDB_TABLE_NAME_LEN - }; +static struct SSchema _s = { + .colId = TSDB_TBNAME_COLUMN_INDEX, + .type = TSDB_DATA_TYPE_BINARY, + .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, + .name = TSQL_TBNAME_L, +}; - strcpy(s.name, TSQL_TBNAME_L); - return s; +SSchema* tGetTbnameColumnSchema() { + return &_s; +} + +static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen) { + int32_t rowLen = 0; + + for (int32_t i = 0; i < numOfCols; ++i) { + // 1. valid types + if (!isValidDataType(pSchema[i].type)) { + return false; + } + + // 2. valid length for each type + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { + if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { + return false; + } + } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { + return false; + } + } else { + if (pSchema[i].bytes != tDataTypes[pSchema[i].type].bytes) { + return false; + } + } + + // 3. valid column names + for (int32_t j = i + 1; j < numOfCols; ++j) { + if (strncasecmp(pSchema[i].name, pSchema[j].name, sizeof(pSchema[i].name) - 1) == 0) { + return false; + } + } + + rowLen += pSchema[i].bytes; + } + + return rowLen <= maxLen; +} + +bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags) { + if (!VALIDNUMOFCOLS(numOfCols)) { + return false; + } + + if (!VALIDNUMOFTAGS(numOfTags)) { + return false; + } + + /* first column must be the timestamp, which is a primary key */ + if (pSchema[0].type != TSDB_DATA_TYPE_TIMESTAMP) { + return false; + } + + if (!doValidateSchema(pSchema, numOfCols, TSDB_MAX_BYTES_PER_ROW)) { + return false; + } + + if (!doValidateSchema(&pSchema[numOfCols], numOfTags, TSDB_MAX_TAGS_LEN)) { + return false; + } + + return true; +} + +int32_t tNameExtractFullName(const SName* name, char* dst) { + assert(name != NULL && dst != NULL); + + // invalid full name format, abort + if (!tIsValidName(name)) { + return -1; + } + + int32_t len = snprintf(dst, TSDB_ACCT_ID_LEN + 1 + TSDB_DB_NAME_LEN, "%s.%s", name->acctId, name->dbname); + + size_t tnameLen = strlen(name->tname); + if (tnameLen > 0) { + assert(name->type == TSDB_TABLE_NAME_T); + dst[len] = TS_PATH_DELIMITER[0]; + + memcpy(dst + len + 1, name->tname, tnameLen); + dst[len + tnameLen + 1] = 0; + } + + return 0; +} + +int32_t tNameLen(const SName* name) { + assert(name != NULL); + int32_t len = (int32_t) strlen(name->acctId); + int32_t len1 = (int32_t) strlen(name->dbname); + int32_t len2 = (int32_t) strlen(name->tname); + + if (name->type == TSDB_DB_NAME_T) { + assert(len2 == 0); + return len + len1 + TS_PATH_DELIMITER_LEN; + } else { + assert(len2 > 0); + return len + len1 + len2 + TS_PATH_DELIMITER_LEN * 2; + } +} + +bool tIsValidName(const SName* name) { + assert(name != NULL); + + if (!VALID_NAME_TYPE(name->type)) { + return false; + } + + if (strlen(name->acctId) <= 0) { + return false; + } + + if (name->type == TSDB_DB_NAME_T) { + return strlen(name->dbname) > 0; + } else { + return strlen(name->dbname) > 0 && strlen(name->tname) > 0; + } +} + +SName* tNameDup(const SName* name) { + assert(name != NULL); + + SName* p = calloc(1, sizeof(SName)); + memcpy(p, name, sizeof(SName)); + return p; +} + +int32_t tNameGetDbName(const SName* name, char* dst) { + assert(name != NULL && dst != NULL); + strncpy(dst, name->dbname, tListLen(name->dbname)); + return 0; +} + +int32_t tNameGetFullDbName(const SName* name, char* dst) { + assert(name != NULL && dst != NULL); + snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN, + "%s.%s", name->acctId, name->dbname); + return 0; +} + +bool tNameIsEmpty(const SName* name) { + assert(name != NULL); + return name->type == 0 || strlen(name->acctId) <= 0; +} + +const char* tNameGetTableName(const SName* name) { + assert(name != NULL && name->type == TSDB_TABLE_NAME_T); + return &name->tname[0]; +} + +void tNameAssign(SName* dst, const SName* src) { + memcpy(dst, src, sizeof(SName)); +} + +int32_t tNameSetDbName(SName* dst, const char* acct, SStrToken* dbToken) { + assert(dst != NULL && dbToken != NULL && acct != NULL); + + // too long account id or too long db name + if (strlen(acct) >= tListLen(dst->acctId) || dbToken->n >= tListLen(dst->dbname)) { + return -1; + } + + dst->type = TSDB_DB_NAME_T; + tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); + tstrncpy(dst->dbname, dbToken->z, dbToken->n + 1); + return 0; +} + +int32_t tNameSetAcctId(SName* dst, const char* acct) { + assert(dst != NULL && acct != NULL); + + // too long account id or too long db name + if (strlen(acct) >= tListLen(dst->acctId)) { + return -1; + } + + tstrncpy(dst->acctId, acct, tListLen(dst->acctId)); + + assert(strlen(dst->acctId) > 0); + + return 0; +} + +int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { + assert(dst != NULL && str != NULL && strlen(str) > 0); + + char* p = NULL; + if ((type & T_NAME_ACCT) == T_NAME_ACCT) { + p = strstr(str, TS_PATH_DELIMITER); + if (p == NULL) { + return -1; + } + + int32_t len = (int32_t)(p - str); + + // too long account id or too long db name + if ((len >= tListLen(dst->acctId)) || (len <= 0)) { + return -1; + } + + memcpy (dst->acctId, str, len); + dst->acctId[len] = 0; + + assert(strlen(dst->acctId) > 0); + } + + if ((type & T_NAME_DB) == T_NAME_DB) { + dst->type = TSDB_DB_NAME_T; + char* start = (char*)((p == NULL)? str:(p+1)); + + int32_t len = 0; + p = strstr(start, TS_PATH_DELIMITER); + if (p == NULL) { + len = (int32_t) strlen(start); + } else { + len = (int32_t) (p - start); + } + + // too long account id or too long db name + if ((len >= tListLen(dst->dbname)) || (len <= 0)) { + return -1; + } + + memcpy (dst->dbname, start, len); + dst->dbname[len] = 0; + } + + if ((type & T_NAME_TABLE) == T_NAME_TABLE) { + dst->type = TSDB_TABLE_NAME_T; + char* start = (char*) ((p == NULL)? str: (p+1)); + + int32_t len = (int32_t) strlen(start); + + // too long account id or too long db name + if ((len >= tListLen(dst->tname)) || (len <= 0)) { + return -1; + } + + memcpy (dst->tname, start, len); + dst->tname[len] = 0; + } + + return 0; } diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 8197fb104282d28795983b88c24dec19661de896..6fa27a029bfd5356cca3e34dffe8d3018ade9fd8 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "taosdef.h" +#include "ttype.h" #include "ttokendef.h" #include "tscompression.h" @@ -367,7 +367,7 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i *maxIndex = 0; } -tDataTypeDescriptor tDataTypeDesc[15] = { +tDataTypeDescriptor tDataTypes[15] = { {TSDB_DATA_TYPE_NULL, 6,1, "NOTYPE", NULL, NULL, NULL}, {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", tsCompressBool, tsDecompressBool, getStatics_bool}, {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, @@ -423,58 +423,58 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { switch (type) { case TSDB_DATA_TYPE_BOOL: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_BOOL_NULL; + *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_BOOL_NULL; } break; case TSDB_DATA_TYPE_TINYINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_TINYINT_NULL; + *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_TINYINT_NULL; } break; case TSDB_DATA_TYPE_SMALLINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint16_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_SMALLINT_NULL; + *(uint16_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_SMALLINT_NULL; } break; case TSDB_DATA_TYPE_INT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_INT_NULL; + *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_INT_NULL; } break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_BIGINT_NULL; + *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_BIGINT_NULL; } break; case TSDB_DATA_TYPE_UTINYINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_UTINYINT_NULL; + *(uint8_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UTINYINT_NULL; } break; case TSDB_DATA_TYPE_USMALLINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint16_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_USMALLINT_NULL; + *(uint16_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_USMALLINT_NULL; } break; case TSDB_DATA_TYPE_UINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_UINT_NULL; + *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UINT_NULL; } break; case TSDB_DATA_TYPE_UBIGINT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_UBIGINT_NULL; + *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_UBIGINT_NULL; } break; case TSDB_DATA_TYPE_FLOAT: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_FLOAT_NULL; + *(uint32_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_FLOAT_NULL; } break; case TSDB_DATA_TYPE_DOUBLE: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL; + *(uint64_t *)(val + i * tDataTypes[type].bytes) = TSDB_DATA_DOUBLE_NULL; } break; case TSDB_DATA_TYPE_NCHAR: @@ -485,7 +485,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { break; default: { for (int32_t i = 0; i < numOfElems; ++i) { - *(uint32_t *)(val + i * tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize) = TSDB_DATA_INT_NULL; + *(uint32_t *)(val + i * tDataTypes[TSDB_DATA_TYPE_INT].bytes) = TSDB_DATA_INT_NULL; } break; } @@ -524,15 +524,18 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: *((int8_t *)val) = GET_INT8_VAL(src); break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: *((int16_t *)val) = GET_INT16_VAL(src); break; - case TSDB_DATA_TYPE_INT: { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: *((int32_t *)val) = GET_INT32_VAL(src); break; - } + case TSDB_DATA_TYPE_FLOAT: SET_FLOAT_VAL(val, GET_FLOAT_VAL(src)); break; @@ -540,6 +543,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { SET_DOUBLE_VAL(val, GET_DOUBLE_VAL(src)); break; case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_TIMESTAMP: *((int64_t *)val) = GET_INT64_VAL(src); break; @@ -628,7 +632,7 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo } // the string may be overflow according to errno - *value = issigned? strtoll(z, &endPtr, radix):strtoul(z, &endPtr, radix); + *value = issigned? strtoll(z, &endPtr, radix):strtoull(z, &endPtr, radix); // not a valid integer number, return error if (endPtr - z != n || errno == ERANGE) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 09c5c761637ee40ea22fc899d517389594c2c98d..7009c4d5c3d8516f17dd4caf89b37e9c28f93274 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -86,43 +86,53 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 switch (type) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: { + pVar->nLen = tDataTypes[type].bytes; pVar->i64 = GET_INT8_VAL(pz); break; } case TSDB_DATA_TYPE_UTINYINT: { + pVar->nLen = tDataTypes[type].bytes; pVar->u64 = GET_UINT8_VAL(pz); break; } case TSDB_DATA_TYPE_SMALLINT: { + pVar->nLen = tDataTypes[type].bytes; pVar->i64 = GET_INT16_VAL(pz); break; } case TSDB_DATA_TYPE_USMALLINT: { + pVar->nLen = tDataTypes[type].bytes; pVar->u64 = GET_UINT16_VAL(pz); break; } case TSDB_DATA_TYPE_INT: { + pVar->nLen = tDataTypes[type].bytes; pVar->i64 = GET_INT32_VAL(pz); break; } case TSDB_DATA_TYPE_UINT: { + pVar->nLen = tDataTypes[type].bytes; pVar->u64 = GET_UINT32_VAL(pz); break; } case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { + pVar->nLen = tDataTypes[type].bytes; pVar->i64 = GET_INT64_VAL(pz); break; } case TSDB_DATA_TYPE_UBIGINT: { + pVar->nLen = tDataTypes[type].bytes; pVar->u64 = GET_UINT64_VAL(pz); break; } case TSDB_DATA_TYPE_DOUBLE: { + pVar->nLen = tDataTypes[type].bytes; pVar->dKey = GET_DOUBLE_VAL(pz); break; } case TSDB_DATA_TYPE_FLOAT: { + pVar->nLen = tDataTypes[type].bytes; pVar->dKey = GET_FLOAT_VAL(pz); break; } @@ -144,6 +154,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 default: pVar->i64 = GET_INT32_VAL(pz); + pVar->nLen = tDataTypes[TSDB_DATA_TYPE_INT].bytes; } pVar->nType = type; @@ -205,7 +216,7 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } if (pDst->nType != TSDB_DATA_TYPE_ARRAY) { - pDst->nLen = tDataTypeDesc[pDst->nType].nSize; + pDst->nLen = tDataTypes[pDst->nType].bytes; } } @@ -425,12 +436,12 @@ static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *val static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result, int32_t type, bool issigned, bool releaseVariantPtr) { if (pVariant->nType == TSDB_DATA_TYPE_NULL) { - setNull((char *)result, type, tDataTypeDesc[type].nSize); + setNull((char *)result, type, tDataTypes[type].bytes); return 0; } errno = 0; - if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType)) { + if (IS_SIGNED_NUMERIC_TYPE(pVariant->nType) || (pVariant->nType == TSDB_DATA_TYPE_BOOL)) { *result = pVariant->i64; } else if (IS_UNSIGNED_NUMERIC_TYPE(pVariant->nType)) { *result = pVariant->u64; @@ -446,7 +457,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result pVariant->nLen = 0; } - setNull((char *)result, type, tDataTypeDesc[type].nSize); + setNull((char *)result, type, tDataTypes[type].bytes); return 0; } @@ -496,7 +507,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result free(pVariant->pz); pVariant->nLen = 0; } - setNull((char *)result, type, tDataTypeDesc[type].nSize); + setNull((char *)result, type, tDataTypes[type].bytes); return 0; } else { int64_t val = wcstoll(pVariant->wpz, &endPtr, 10); @@ -775,7 +786,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu return -1; } } else { - wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen); + memcpy(p, pVariant->wpz, pVariant->nLen); newlen = pVariant->nLen; } @@ -867,4 +878,4 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) { } return 0; -} \ No newline at end of file +} diff --git a/src/connector/jdbc/.classpath b/src/connector/jdbc/.classpath deleted file mode 100644 index a5d95095ccaaf9549ee22ff2fbf684bfa43d31c9..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/.classpath +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/connector/jdbc/.project b/src/connector/jdbc/.project deleted file mode 100644 index 656ab58d205bf0e01c151728a75e5ab543facee6..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/.project +++ /dev/null @@ -1,23 +0,0 @@ - - - taos-jdbcdriver - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - - diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 22d3d31547d0653a0ce0afe7b4128219fef35098..eb158b1f769f97482a314f2797ec247f756b3b39 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.15-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.25-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 1a86bc57dcc86c45ef5deb05fc58b39ee0939059..eb8c92575c4b9953e5b8155d2bf2b1ded4772a26 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.15 + 2.0.25 jar JDBCDriver @@ -37,17 +37,6 @@ - - commons-logging - commons-logging - 1.2 - - - * - * - - - junit junit @@ -61,21 +50,20 @@ httpclient 4.5.8 - - org.apache.commons - commons-lang3 - 3.9 - com.alibaba fastjson 1.2.58 + + com.google.guava + guava + 29.0-jre + - org.apache.maven.plugins maven-assembly-plugin diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 9865fc7127b39debc965f6be9bad6e8485169416..1f75754b0c46ec0afced88743c5aba1d308c24ec 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.15 + 2.0.25 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -34,60 +34,42 @@ UTF-8 1.8 3.6.0 - 1.1.2 - 3.5 + - - commons-logging - commons-logging - ${commons-logging.version} - - - * - * - - - junit junit 4.13 test - - - mysql - mysql-connector-java - 5.1.47 - - org.apache.httpcomponents httpclient 4.5.8 - - org.apache.commons - commons-lang3 - 3.9 - com.alibaba fastjson 1.2.58 - - mysql - mysql-connector-java - 5.1.49 + com.google.guava + guava + 29.0-jre - + + + src/main/resources + + **/*.md + + + org.apache.maven.plugins @@ -125,12 +107,21 @@ maven-surefire-plugin 2.12.4 + pertest + ${maven.test.jvmargs} **/*Test.java - **/BatchInsertTest.java + **/TSDBJNIConnectorTest.java + **/UnsignedNumberJniTest.java + **/DatetimeBefore1970Test.java + **/AppMemoryLeakTest.java + **/AuthenticationTest.java + **/TaosInfoMonitorTest.java **/FailOverTest.java + **/InvalidResultSetPointerTest.java + **/RestfulConnectionTest.java true diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java new file mode 100644 index 0000000000000000000000000000000000000000..976078da95c387110a2e5e55076a42a2c320ca18 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractConnection.java @@ -0,0 +1,532 @@ +package com.taosdata.jdbc; + +import java.sql.*; +import java.util.Enumeration; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.*; + +public abstract class AbstractConnection extends WrapperImpl implements Connection { + + protected volatile boolean isClosed; + protected volatile String catalog; + protected volatile Properties clientInfoProps = new Properties(); + + @Override + public abstract Statement createStatement() throws SQLException; + + @Override + public abstract PreparedStatement prepareStatement(String sql) throws SQLException; + + @Override + public CallableStatement prepareCall(String sql) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public String nativeSQL(String sql) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + // do nothing + + return sql; + } + + + + @Override + public void setAutoCommit(boolean autoCommit) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + //do nothing + } + + @Override + public boolean getAutoCommit() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return true; + } + + @Override + public void commit() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + // do nothing + } + + @Override + public void rollback() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + // do nothing + } + + @Override + public abstract void close() throws SQLException; + + @Override + public abstract boolean isClosed() throws SQLException; + + @Override + public abstract DatabaseMetaData getMetaData() throws SQLException; + + @Override + public void setReadOnly(boolean readOnly) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + //do nothing + } + + @Override + public boolean isReadOnly() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return true; + } + + @Override + public void setCatalog(String catalog) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + /* + try (Statement stmt = createStatement()) { + boolean execute = stmt.execute("use " + catalog); + if (execute) + this.catalog = catalog; + } catch (SQLException e) { + // do nothing + } + */ + + this.catalog = catalog; + } + + @Override + public String getCatalog() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return this.catalog; + } + + @Override + public void setTransactionIsolation(int level) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (level) { + case Connection.TRANSACTION_NONE: + break; + case Connection.TRANSACTION_READ_UNCOMMITTED: + case Connection.TRANSACTION_READ_COMMITTED: + case Connection.TRANSACTION_REPEATABLE_READ: + case Connection.TRANSACTION_SERIALIZABLE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + //do nothing + } + + @Override + public int getTransactionIsolation() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return Connection.TRANSACTION_NONE; + } + + @Override + public SQLWarning getWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return null; + } + + @Override + public void clearWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + // do nothing + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (resultSetType) { + case ResultSet.TYPE_FORWARD_ONLY: + break; + case ResultSet.TYPE_SCROLL_INSENSITIVE: + case ResultSet.TYPE_SCROLL_SENSITIVE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + switch (resultSetConcurrency) { + case ResultSet.CONCUR_READ_ONLY: + break; + case ResultSet.CONCUR_UPDATABLE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + return createStatement(); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (resultSetType) { + case ResultSet.TYPE_FORWARD_ONLY: + break; + case ResultSet.TYPE_SCROLL_INSENSITIVE: + case ResultSet.TYPE_SCROLL_SENSITIVE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + switch (resultSetConcurrency) { + case ResultSet.CONCUR_READ_ONLY: + break; + case ResultSet.CONCUR_UPDATABLE: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + return prepareStatement(sql); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Map> getTypeMap() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTypeMap(Map> map) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setHoldability(int holdability) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (holdability) { + case ResultSet.HOLD_CURSORS_OVER_COMMIT: + break; + case ResultSet.CLOSE_CURSORS_AT_COMMIT: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + //do nothing + } + + @Override + public int getHoldability() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public Savepoint setSavepoint() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Savepoint setSavepoint(String name) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void rollback(Savepoint savepoint) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void releaseSavepoint(Savepoint savepoint) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (resultSetHoldability) { + case ResultSet.HOLD_CURSORS_OVER_COMMIT: + break; + case ResultSet.CLOSE_CURSORS_AT_COMMIT: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + return createStatement(resultSetType, resultSetConcurrency); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (resultSetHoldability) { + case ResultSet.HOLD_CURSORS_OVER_COMMIT: + break; + case ResultSet.CLOSE_CURSORS_AT_COMMIT: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + return prepareStatement(sql, resultSetType, resultSetConcurrency); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + switch (autoGeneratedKeys) { + case Statement.RETURN_GENERATED_KEYS: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + case Statement.NO_GENERATED_KEYS: + break; + } + return prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Override + public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Clob createClob() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Blob createBlob() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public NClob createNClob() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public SQLXML createSQLXML() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean isValid(int timeout) throws SQLException { + //true if the connection is valid, false otherwise + if (isClosed()) + return false; + if (timeout < 0) //SQLException - if the value supplied for timeout is less then 0 + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + + ExecutorService executor = Executors.newCachedThreadPool(); + Future future = executor.submit(() -> { + int status; + try (Statement stmt = createStatement()) { + ResultSet resultSet = stmt.executeQuery("select server_status()"); + resultSet.next(); + status = resultSet.getInt("server_status()"); + resultSet.close(); + } + return status == 1 ? true : false; + }); + + boolean status = false; + try { + if (timeout == 0) + status = future.get(); + else + status = future.get(timeout, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } catch (TimeoutException e) { + future.cancel(true); + status = false; + } finally { + executor.shutdownNow(); + } + return status; + } + + @Override + public void setClientInfo(String name, String value) throws SQLClientInfoException { + if (isClosed) + throw (SQLClientInfoException) TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); + + if (clientInfoProps == null) + clientInfoProps = new Properties(); + clientInfoProps.setProperty(name, value); + } + + @Override + public void setClientInfo(Properties properties) throws SQLClientInfoException { + if (isClosed) + throw (SQLClientInfoException) TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); + + for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { + String name = (String) enumer.nextElement(); + clientInfoProps.put(name, properties.getProperty(name)); + } + } + + @Override + public String getClientInfo(String name) throws SQLException { + if (isClosed) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return clientInfoProps.getProperty(name); + } + + @Override + public Properties getClientInfo() throws SQLException { + if (isClosed) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return clientInfoProps; + } + + @Override + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setSchema(String schema) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + //do nothing + } + + @Override + public String getSchema() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + return null; + } + + @Override + public void abort(Executor executor) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + // do nothing + } + + @Override + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + if (milliseconds < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int getNetworkTimeout() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java index 1445be18654ff3e73b74484b47e09856ddc94b01..5dcaa77ebd4a15087785a6a9b642b85f160f5287 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDatabaseMetaData.java @@ -1,28 +1,13 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ package com.taosdata.jdbc; import java.sql.*; import java.util.ArrayList; import java.util.List; -public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { +public abstract class AbstractDatabaseMetaData extends WrapperImpl implements DatabaseMetaData { private final static String PRODUCT_NAME = "TDengine"; private final static String PRODUCT_VESION = "2.0.x.x"; - private final static String DRIVER_NAME = "taos-jdbcdriver"; private final static String DRIVER_VERSION = "2.0.x"; private final static int DRIVER_MAJAR_VERSION = 2; private final static int DRIVER_MINOR_VERSION = 0; @@ -67,9 +52,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { return PRODUCT_VESION; } - public String getDriverName() throws SQLException { - return DRIVER_NAME; - } + public abstract String getDriverName() throws SQLException; public String getDriverVersion() throws SQLException { return DRIVER_VERSION; @@ -92,6 +75,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public boolean supportsMixedCaseIdentifiers() throws SQLException { + //像database、table这些对象的标识符,在存储时是否采用大小写混合的模式 return false; } @@ -100,7 +84,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public boolean storesLowerCaseIdentifiers() throws SQLException { - return false; + return true; } public boolean storesMixedCaseIdentifiers() throws SQLException { @@ -168,10 +152,12 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public boolean nullPlusNonNullIsNull() throws SQLException { + // null + non-null != null return false; } public boolean supportsConvert() throws SQLException { + // 是否支持转换函数convert return false; } @@ -196,7 +182,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public boolean supportsGroupBy() throws SQLException { - return false; + return true; } public boolean supportsGroupByUnrelated() throws SQLException { @@ -468,7 +454,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public int getDefaultTransactionIsolation() throws SQLException { - return 0; + return Connection.TRANSACTION_NONE; } public boolean supportsTransactions() throws SQLException { @@ -476,6 +462,8 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public boolean supportsTransactionIsolationLevel(int level) throws SQLException { + if (level == Connection.TRANSACTION_NONE) + return true; return false; } @@ -495,18 +483,113 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { return false; } - public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) - throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) throws SQLException { + return null; } - public ResultSet getProcedureColumns(String catalog, String schemaPattern, String procedureNamePattern, - String columnNamePattern) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + public ResultSet getProcedureColumns(String catalog, String schemaPattern, String procedureNamePattern, String columnNamePattern) throws SQLException { + return null; } - public abstract ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) - throws SQLException; + public abstract ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException; + + protected ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types, Connection connection) throws SQLException { + try (Statement stmt = connection.createStatement()) { + if (catalog == null || catalog.isEmpty()) + return null; + + ResultSet databases = stmt.executeQuery("show databases"); + String dbname = null; + while (databases.next()) { + dbname = databases.getString("name"); + if (dbname.equalsIgnoreCase(catalog)) + break; + } + databases.close(); + if (dbname == null) + return null; + + stmt.execute("use " + dbname); + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + List columnMetaDataList = new ArrayList<>(); + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(1); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + ColumnMetaData col2 = new ColumnMetaData(); + col2.setColIndex(2); + col2.setColName("TABLE_SCHEM"); + col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col2); + ColumnMetaData col3 = new ColumnMetaData(); + col3.setColIndex(3); + col3.setColName("TABLE_NAME"); + col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col3); + ColumnMetaData col4 = new ColumnMetaData(); + col4.setColIndex(4); + col4.setColName("TABLE_TYPE"); + col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col4); + ColumnMetaData col5 = new ColumnMetaData(); + col5.setColIndex(5); + col5.setColName("REMARKS"); + col5.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col5); + ColumnMetaData col6 = new ColumnMetaData(); + col6.setColIndex(6); + col6.setColName("TYPE_CAT"); + col6.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col6); + ColumnMetaData col7 = new ColumnMetaData(); + col7.setColIndex(7); + col7.setColName("TYPE_SCHEM"); + col7.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col7); + ColumnMetaData col8 = new ColumnMetaData(); + col8.setColIndex(8); + col8.setColName("TYPE_NAME"); + col8.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col8); + ColumnMetaData col9 = new ColumnMetaData(); + col9.setColIndex(9); + col9.setColName("SELF_REFERENCING_COL_NAME"); + col9.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col9); + ColumnMetaData col10 = new ColumnMetaData(); + col10.setColIndex(10); + col10.setColName("REF_GENERATION"); + col10.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col10); + resultSet.setColumnMetaDataList(columnMetaDataList); + + List rowDataList = new ArrayList<>(); + ResultSet tables = stmt.executeQuery("show tables"); + while (tables.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(10); + rowData.setString(0, dbname); //table_cat + rowData.setString(1, null); //TABLE_SCHEM + rowData.setString(2, tables.getString("table_name")); //TABLE_NAME + rowData.setString(3, "TABLE"); //TABLE_TYPE + rowData.setString(4, ""); //REMARKS + rowDataList.add(rowData); + } + + ResultSet stables = stmt.executeQuery("show stables"); + while (stables.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(10); + rowData.setString(0, dbname); //TABLE_CAT + rowData.setString(1, null); //TABLE_SCHEM + rowData.setString(2, stables.getString("name")); //TABLE_NAME + rowData.setString(3, "TABLE"); //TABLE_TYPE + rowData.setString(4, "STABLE"); //REMARKS + rowDataList.add(rowData); + } + resultSet.setRowDataList(rowDataList); + return resultSet; + } + } public ResultSet getSchemas() throws SQLException { return getEmptyResultSet(); @@ -516,32 +599,239 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { public ResultSet getTableTypes() throws SQLException { DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); - // set up ColumnMetaDataList - List columnMetaDataList = new ArrayList(1); + List columnMetaDataList = new ArrayList<>(); ColumnMetaData colMetaData = new ColumnMetaData(); colMetaData.setColIndex(0); colMetaData.setColName("TABLE_TYPE"); colMetaData.setColSize(10); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); + colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); columnMetaDataList.add(colMetaData); + resultSet.setColumnMetaDataList(columnMetaDataList); // set up rowDataList - List rowDataList = new ArrayList(2); - TSDBResultSetRowData rowData = new TSDBResultSetRowData(); + List rowDataList = new ArrayList<>(); + TSDBResultSetRowData rowData = new TSDBResultSetRowData(1); rowData.setString(0, "TABLE"); rowDataList.add(rowData); - rowData = new TSDBResultSetRowData(); + rowData = new TSDBResultSetRowData(1); rowData.setString(0, "STABLE"); rowDataList.add(rowData); - - resultSet.setColumnMetaDataList(columnMetaDataList); resultSet.setRowDataList(rowDataList); + return resultSet; } public abstract ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException; + protected ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern, Connection conn) { + try (Statement stmt = conn.createStatement()) { + if (catalog == null || catalog.isEmpty()) + return null; + + ResultSet databases = stmt.executeQuery("show databases"); + String dbname = null; + while (databases.next()) { + dbname = databases.getString("name"); + if (dbname.equalsIgnoreCase(catalog)) + break; + } + databases.close(); + if (dbname == null) + return null; + + stmt.execute("use " + dbname); + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + + List columnMetaDataList = new ArrayList<>(); + // TABLE_CAT + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(1); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + // TABLE_SCHEM + ColumnMetaData col2 = new ColumnMetaData(); + col2.setColIndex(2); + col2.setColName("TABLE_SCHEM"); + col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col2); + // TABLE_NAME + ColumnMetaData col3 = new ColumnMetaData(); + col3.setColIndex(3); + col3.setColName("TABLE_NAME"); + col3.setColSize(193); + col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col3); + // COLUMN_NAME + ColumnMetaData col4 = new ColumnMetaData(); + col4.setColIndex(4); + col4.setColName("COLUMN_NAME"); + col4.setColSize(65); + col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col4); + // DATA_TYPE + ColumnMetaData col5 = new ColumnMetaData(); + col5.setColIndex(5); + col5.setColName("DATA_TYPE"); + col5.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col5); + // TYPE_NAME + ColumnMetaData col6 = new ColumnMetaData(); + col6.setColIndex(6); + col6.setColName("TYPE_NAME"); + col6.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col6); + // COLUMN_SIZE + ColumnMetaData col7 = new ColumnMetaData(); + col7.setColIndex(7); + col7.setColName("COLUMN_SIZE"); + col7.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col7); + // BUFFER_LENGTH, not used + ColumnMetaData col8 = new ColumnMetaData(); + col8.setColIndex(8); + col8.setColName("BUFFER_LENGTH"); + columnMetaDataList.add(col8); + // DECIMAL_DIGITS + ColumnMetaData col9 = new ColumnMetaData(); + col9.setColIndex(9); + col9.setColName("DECIMAL_DIGITS"); + col9.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col9); + // add NUM_PREC_RADIX + ColumnMetaData col10 = new ColumnMetaData(); + col10.setColIndex(10); + col10.setColName("NUM_PREC_RADIX"); + col10.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col10); + // NULLABLE + ColumnMetaData col11 = new ColumnMetaData(); + col11.setColIndex(11); + col11.setColName("NULLABLE"); + col11.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col11); + // REMARKS + ColumnMetaData col12 = new ColumnMetaData(); + col12.setColIndex(12); + col12.setColName("REMARKS"); + col12.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col12); + // COLUMN_DEF + ColumnMetaData col13 = new ColumnMetaData(); + col13.setColIndex(13); + col13.setColName("COLUMN_DEF"); + col13.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col13); + //SQL_DATA_TYPE + ColumnMetaData col14 = new ColumnMetaData(); + col14.setColIndex(14); + col14.setColName("SQL_DATA_TYPE"); + col14.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col14); + //SQL_DATETIME_SUB + ColumnMetaData col15 = new ColumnMetaData(); + col15.setColIndex(15); + col15.setColName("SQL_DATETIME_SUB"); + col15.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col15); + //CHAR_OCTET_LENGTH + ColumnMetaData col16 = new ColumnMetaData(); + col16.setColIndex(16); + col16.setColName("CHAR_OCTET_LENGTH"); + col16.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col16); + //ORDINAL_POSITION + ColumnMetaData col17 = new ColumnMetaData(); + col17.setColIndex(17); + col17.setColName("ORDINAL_POSITION"); + col17.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col17); + // IS_NULLABLE + ColumnMetaData col18 = new ColumnMetaData(); + col18.setColIndex(18); + col18.setColName("IS_NULLABLE"); + col18.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col18); + //SCOPE_CATALOG + ColumnMetaData col19 = new ColumnMetaData(); + col19.setColIndex(19); + col19.setColName("SCOPE_CATALOG"); + col19.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col19); + //SCOPE_SCHEMA + ColumnMetaData col20 = new ColumnMetaData(); + col20.setColIndex(20); + col20.setColName("SCOPE_SCHEMA"); + col20.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col20); + //SCOPE_TABLE + ColumnMetaData col21 = new ColumnMetaData(); + col21.setColIndex(21); + col21.setColName("SCOPE_TABLE"); + col21.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col21); + //SOURCE_DATA_TYPE + ColumnMetaData col22 = new ColumnMetaData(); + col22.setColIndex(22); + col22.setColName("SOURCE_DATA_TYPE"); + col22.setColType(TSDBConstants.TSDB_DATA_TYPE_SMALLINT); + columnMetaDataList.add(col22); + //IS_AUTOINCREMENT + ColumnMetaData col23 = new ColumnMetaData(); + col23.setColIndex(23); + col23.setColName("IS_AUTOINCREMENT"); + col23.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col23); + //IS_GENERATEDCOLUMN + ColumnMetaData col24 = new ColumnMetaData(); + col24.setColIndex(24); + col24.setColName("IS_GENERATEDCOLUMN"); + col24.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col24); + + resultSet.setColumnMetaDataList(columnMetaDataList); + // set up rowDataList + ResultSet rs = stmt.executeQuery("describe " + dbname + "." + tableNamePattern); + List rowDataList = new ArrayList<>(); + int index = 0; + while (rs.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(24); + // set TABLE_CAT + rowData.setString(0, dbname); + // set TABLE_NAME + rowData.setString(2, tableNamePattern); + // set COLUMN_NAME + rowData.setString(3, rs.getString("Field")); + // set DATA_TYPE + String typeName = rs.getString("Type"); + rowData.setInt(4, getDataType(typeName)); + // set TYPE_NAME + rowData.setString(5, typeName); + // set COLUMN_SIZE + int length = rs.getInt("Length"); + rowData.setInt(6, getColumnSize(typeName, length)); + // set DECIMAL_DIGITS + rowData.setInt(8, getDecimalDigits(typeName)); + // set NUM_PREC_RADIX + rowData.setInt(9, 10); + // set NULLABLE + rowData.setInt(10, getNullable(index, typeName)); + // set REMARKS + rowData.setString(11, rs.getString("Note")); + rowDataList.add(rowData); + index++; + } + resultSet.setRowDataList(rowDataList); + return resultSet; + + } catch (SQLException e) { + e.printStackTrace(); + } + return null; + } + protected int getNullable(int index, String typeName) { if (index == 0 && "TIMESTAMP".equals(typeName)) return DatabaseMetaData.columnNoNulls; @@ -552,7 +842,6 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { switch (typeName) { case "TIMESTAMP": return 23; - default: return 0; } @@ -615,9 +904,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { return getEmptyResultSet(); } - public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); - } + public abstract ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException; public ResultSet getImportedKeys(String catalog, String schema, String table) throws SQLException { return getEmptyResultSet(); @@ -694,9 +981,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { return getEmptyResultSet(); } - public Connection getConnection() throws SQLException { - return null; - } + public abstract Connection getConnection() throws SQLException; public boolean supportsSavepoints() throws SQLException { return false; @@ -718,9 +1003,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { return getEmptyResultSet(); } - public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { - return getEmptyResultSet(); - } + public abstract ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException; public ResultSet getAttributes(String catalog, String schemaPattern, String typeNamePattern, String attributeNamePattern) throws SQLException { @@ -728,15 +1011,17 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public boolean supportsResultSetHoldability(int holdability) throws SQLException { + if (holdability == ResultSet.HOLD_CURSORS_OVER_COMMIT) + return true; return false; } public int getResultSetHoldability() throws SQLException { - return 0; + return ResultSet.HOLD_CURSORS_OVER_COMMIT; } public int getDatabaseMajorVersion() throws SQLException { - return 0; + return 2; } public int getDatabaseMinorVersion() throws SQLException { @@ -744,7 +1029,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public int getJDBCMajorVersion() throws SQLException { - return 0; + return 2; } public int getJDBCMinorVersion() throws SQLException { @@ -768,7 +1053,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { - return null; + return getEmptyResultSet(); } public boolean supportsStoredFunctionsUsingCallSyntax() throws SQLException { @@ -780,6 +1065,7 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { } public ResultSet getClientInfoProperties() throws SQLException { + //TODO: see https://docs.oracle.com/javase/8/docs/api/java/sql/Connection.html#setClientInfo-java.lang.String-java.lang.String- return getEmptyResultSet(); } @@ -805,4 +1091,166 @@ public abstract class AbstractDatabaseMetaData implements DatabaseMetaData { private ResultSet getEmptyResultSet() { return new EmptyResultSet(); } + + protected ResultSet getCatalogs(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + List columnMetaDataList = new ArrayList<>(); + // TABLE_CAT + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(1); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + + resultSet.setColumnMetaDataList(columnMetaDataList); + + List rowDataList = new ArrayList<>(); + ResultSet rs = stmt.executeQuery("show databases"); + while (rs.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(1); + rowData.setString(0, rs.getString("name")); + rowDataList.add(rowData); + } + resultSet.setRowDataList(rowDataList); + return resultSet; + } + } + + + protected ResultSet getPrimaryKeys(String catalog, String schema, String table, Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + if (catalog == null || catalog.isEmpty()) + return null; + + ResultSet databases = stmt.executeQuery("show databases"); + String dbname = null; + while (databases.next()) { + dbname = databases.getString("name"); + if (dbname.equalsIgnoreCase(catalog)) + break; + } + databases.close(); + if (dbname == null) + return null; + + stmt.execute("use " + dbname); + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + List columnMetaDataList = new ArrayList<>(); + // TABLE_CAT + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(0); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + // TABLE_SCHEM + ColumnMetaData col2 = new ColumnMetaData(); + col2.setColIndex(1); + col2.setColName("TABLE_SCHEM"); + col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col2); + // TABLE_NAME + ColumnMetaData col3 = new ColumnMetaData(); + col3.setColIndex(2); + col3.setColName("TABLE_NAME"); + col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col3); + // COLUMN_NAME + ColumnMetaData col4 = new ColumnMetaData(); + col4.setColIndex(3); + col4.setColName("COLUMN_NAME"); + col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col4); + // KEY_SEQ + ColumnMetaData col5 = new ColumnMetaData(); + col5.setColIndex(4); + col5.setColName("KEY_SEQ"); + col5.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); + columnMetaDataList.add(col5); + // PK_NAME + ColumnMetaData col6 = new ColumnMetaData(); + col6.setColIndex(5); + col6.setColName("PK_NAME"); + col6.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col6); + resultSet.setColumnMetaDataList(columnMetaDataList); + + // set rowData + List rowDataList = new ArrayList<>(); + ResultSet rs = stmt.executeQuery("describe " + dbname + "." + table); + rs.next(); + TSDBResultSetRowData rowData = new TSDBResultSetRowData(6); + rowData.setString(0, null); + rowData.setString(1, null); + rowData.setString(2, table); + String pkName = rs.getString(1); + rowData.setString(3, pkName); + rowData.setInt(4, 1); + rowData.setString(5, pkName); + rowDataList.add(rowData); + resultSet.setRowDataList(rowDataList); + return resultSet; + } + } + + protected ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern, Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + if (catalog == null || catalog.isEmpty()) + return null; + + ResultSet databases = stmt.executeQuery("show databases"); + String dbname = null; + while (databases.next()) { + dbname = databases.getString("name"); + if (dbname.equalsIgnoreCase(catalog)) + break; + } + databases.close(); + if (dbname == null) + return null; + + stmt.execute("use " + dbname); + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + // set up ColumnMetaDataList + List columnMetaDataList = new ArrayList<>(); + // TABLE_CAT + ColumnMetaData col1 = new ColumnMetaData(); + col1.setColIndex(0); + col1.setColName("TABLE_CAT"); + col1.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col1); + // TABLE_SCHEM + ColumnMetaData col2 = new ColumnMetaData(); + col2.setColIndex(1); + col2.setColName("TABLE_SCHEM"); + col2.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col2); + // TABLE_NAME + ColumnMetaData col3 = new ColumnMetaData(); + col3.setColIndex(2); + col3.setColName("TABLE_NAME"); + col3.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col3); + // SUPERTABLE_NAME + ColumnMetaData col4 = new ColumnMetaData(); + col4.setColIndex(3); + col4.setColName("SUPERTABLE_NAME"); + col4.setColType(TSDBConstants.TSDB_DATA_TYPE_NCHAR); + columnMetaDataList.add(col4); + resultSet.setColumnMetaDataList(columnMetaDataList); + + ResultSet rs = stmt.executeQuery("show tables like '" + tableNamePattern + "'"); + List rowDataList = new ArrayList<>(); + while (rs.next()) { + TSDBResultSetRowData rowData = new TSDBResultSetRowData(4); + rowData.setString(2, rs.getString(1)); + rowData.setString(3, rs.getString(4)); + rowDataList.add(rowData); + } + resultSet.setRowDataList(rowDataList); + return resultSet; + } + } } \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractTaosDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java similarity index 58% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractTaosDriver.java rename to src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java index f864788bfffc8bdfefb0b91ec645a10ae8eec843..5eaac1cd3ba7283b019a1d294c1a33334a3d9fa7 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractTaosDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractDriver.java @@ -1,82 +1,18 @@ package com.taosdata.jdbc; -import java.io.*; import java.sql.Driver; import java.sql.DriverPropertyInfo; -import java.util.ArrayList; -import java.util.List; import java.util.Properties; import java.util.StringTokenizer; -public abstract class AbstractTaosDriver implements Driver { - - private static final String TAOS_CFG_FILENAME = "taos.cfg"; - - /** - * @param cfgDirPath - * @return return the config dir - **/ - protected File loadConfigDir(String cfgDirPath) { - if (cfgDirPath == null) - return loadDefaultConfigDir(); - File cfgDir = new File(cfgDirPath); - if (!cfgDir.exists()) - return loadDefaultConfigDir(); - return cfgDir; - } - - /** - * @return search the default config dir, if the config dir is not exist will return null - */ - protected File loadDefaultConfigDir() { - File cfgDir; - File cfgDir_linux = new File("/etc/taos"); - cfgDir = cfgDir_linux.exists() ? cfgDir_linux : null; - File cfgDir_windows = new File("C:\\TDengine\\cfg"); - cfgDir = (cfgDir == null && cfgDir_windows.exists()) ? cfgDir_windows : cfgDir; - return cfgDir; - } - - protected List loadConfigEndpoints(File cfgFile) { - List endpoints = new ArrayList<>(); - try (BufferedReader reader = new BufferedReader(new FileReader(cfgFile))) { - String line = null; - while ((line = reader.readLine()) != null) { - if (line.trim().startsWith("firstEp") || line.trim().startsWith("secondEp")) { - endpoints.add(line.substring(line.indexOf('p') + 1).trim()); - } - if (endpoints.size() > 1) - break; - } - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - return endpoints; - } - - protected void loadTaosConfig(Properties info) { - if ((info.getProperty(TSDBDriver.PROPERTY_KEY_HOST) == null || - info.getProperty(TSDBDriver.PROPERTY_KEY_HOST).isEmpty()) && ( - info.getProperty(TSDBDriver.PROPERTY_KEY_PORT) == null || - info.getProperty(TSDBDriver.PROPERTY_KEY_PORT).isEmpty())) { - File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR)); - File cfgFile = cfgDir.listFiles((dir, name) -> TAOS_CFG_FILENAME.equalsIgnoreCase(name))[0]; - List endpoints = loadConfigEndpoints(cfgFile); - if (!endpoints.isEmpty()) { - info.setProperty(TSDBDriver.PROPERTY_KEY_HOST, endpoints.get(0).split(":")[0]); - info.setProperty(TSDBDriver.PROPERTY_KEY_PORT, endpoints.get(0).split(":")[1]); - } - } - } +public abstract class AbstractDriver implements Driver { protected DriverPropertyInfo[] getPropertyInfo(Properties info) { DriverPropertyInfo hostProp = new DriverPropertyInfo(TSDBDriver.PROPERTY_KEY_HOST, info.getProperty(TSDBDriver.PROPERTY_KEY_HOST)); hostProp.required = false; hostProp.description = "Hostname"; - DriverPropertyInfo portProp = new DriverPropertyInfo(TSDBDriver.PROPERTY_KEY_PORT, info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, TSDBConstants.DEFAULT_PORT)); + DriverPropertyInfo portProp = new DriverPropertyInfo(TSDBDriver.PROPERTY_KEY_PORT, info.getProperty(TSDBDriver.PROPERTY_KEY_PORT)); portProp.required = false; portProp.description = "Port"; @@ -104,11 +40,11 @@ public abstract class AbstractTaosDriver implements Driver { protected Properties parseURL(String url, Properties defaults) { Properties urlProps = (defaults != null) ? defaults : new Properties(); - // parse properties + // parse properties in url int beginningOfSlashes = url.indexOf("//"); int index = url.indexOf("?"); if (index != -1) { - String paramString = url.substring(index + 1, url.length()); + String paramString = url.substring(index + 1); url = url.substring(0, index); StringTokenizer queryParams = new StringTokenizer(paramString, "&"); while (queryParams.hasMoreElements()) { @@ -132,6 +68,7 @@ public abstract class AbstractTaosDriver implements Driver { String dbProductName = url.substring(0, beginningOfSlashes); dbProductName = dbProductName.substring(dbProductName.indexOf(":") + 1); dbProductName = dbProductName.substring(0, dbProductName.indexOf(":")); + urlProps.setProperty(TSDBDriver.PROPERTY_KEY_PRODUCT_NAME,dbProductName); // parse dbname url = url.substring(beginningOfSlashes + 2); int indexOfSlash = url.indexOf("/"); @@ -156,6 +93,4 @@ public abstract class AbstractTaosDriver implements Driver { return urlProps; } - - } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java new file mode 100644 index 0000000000000000000000000000000000000000..7df7252ae2010b5aa5e8f74de2cca55844e25b83 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java @@ -0,0 +1,138 @@ +package com.taosdata.jdbc; + +import java.sql.ParameterMetaData; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; + +public abstract class AbstractParameterMetaData extends WrapperImpl implements ParameterMetaData { + + private final Object[] parameters; + + public AbstractParameterMetaData(Object[] parameters) { + this.parameters = parameters; + } + + @Override + public int getParameterCount() throws SQLException { + return parameters == null ? 0 : parameters.length; + } + + @Override + public int isNullable(int param) throws SQLException { + return ParameterMetaData.parameterNullableUnknown; + } + + @Override + public boolean isSigned(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (parameters[param - 1] instanceof Byte) + return true; + if (parameters[param - 1] instanceof Short) + return true; + if (parameters[param - 1] instanceof Integer) + return true; + if (parameters[param - 1] instanceof Long) + return true; + if (parameters[param - 1] instanceof Float) + return true; + if (parameters[param - 1] instanceof Double) + return true; + + return false; + } + + @Override + public int getPrecision(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (parameters[param - 1] instanceof String) + return ((String) parameters[param - 1]).length(); + if (parameters[param - 1] instanceof byte[]) + return ((byte[]) parameters[param - 1]).length; + return 0; + } + + @Override + public int getScale(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + return 0; + } + + @Override + public int getParameterType(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (parameters[param - 1] instanceof Timestamp) + return Types.TIMESTAMP; + if (parameters[param - 1] instanceof Byte) + return Types.TINYINT; + if (parameters[param - 1] instanceof Short) + return Types.SMALLINT; + if (parameters[param - 1] instanceof Integer) + return Types.INTEGER; + if (parameters[param - 1] instanceof Long) + return Types.BIGINT; + if (parameters[param - 1] instanceof Float) + return Types.FLOAT; + if (parameters[param - 1] instanceof Double) + return Types.DOUBLE; + if (parameters[param - 1] instanceof String) + return Types.NCHAR; + if (parameters[param - 1] instanceof byte[]) + return Types.BINARY; + if (parameters[param - 1] instanceof Boolean) + return Types.BOOLEAN; + return Types.OTHER; + } + + @Override + public String getParameterTypeName(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (parameters[param - 1] instanceof Timestamp) + return TSDBConstants.jdbcType2TaosTypeName(Types.TIMESTAMP); + if (parameters[param - 1] instanceof Byte) + return TSDBConstants.jdbcType2TaosTypeName(Types.TINYINT); + if (parameters[param - 1] instanceof Short) + return TSDBConstants.jdbcType2TaosTypeName(Types.SMALLINT); + if (parameters[param - 1] instanceof Integer) + return TSDBConstants.jdbcType2TaosTypeName(Types.INTEGER); + if (parameters[param - 1] instanceof Long) + return TSDBConstants.jdbcType2TaosTypeName(Types.BIGINT); + if (parameters[param - 1] instanceof Float) + return TSDBConstants.jdbcType2TaosTypeName(Types.FLOAT); + if (parameters[param - 1] instanceof Double) + return TSDBConstants.jdbcType2TaosTypeName(Types.DOUBLE); + if (parameters[param - 1] instanceof String) + return TSDBConstants.jdbcType2TaosTypeName(Types.NCHAR); + if (parameters[param - 1] instanceof byte[]) + return TSDBConstants.jdbcType2TaosTypeName(Types.BINARY); + if (parameters[param - 1] instanceof Boolean) + return TSDBConstants.jdbcType2TaosTypeName(Types.BOOLEAN); + + return parameters[param - 1].getClass().getName(); + } + + @Override + public String getParameterClassName(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + return parameters[param - 1].getClass().getName(); + } + + @Override + public int getParameterMode(int param) throws SQLException { + if (param < 1 && param >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + return ParameterMetaData.parameterModeUnknown; + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java new file mode 100644 index 0000000000000000000000000000000000000000..abd348e68c3ebac0d940b4c9789a85ddd68e238e --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractResultSet.java @@ -0,0 +1,1174 @@ +package com.taosdata.jdbc; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.sql.*; +import java.util.Calendar; +import java.util.Map; + +public abstract class AbstractResultSet extends WrapperImpl implements ResultSet { + private int fetchSize; + + protected void checkAvailability(int columnIndex, int bounds) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + if (columnIndex < 1) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " < 1"); + if (columnIndex > bounds) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "Column Index out of range, " + columnIndex + " > " + bounds); + } + + @Override + public abstract boolean next() throws SQLException; + + @Override + public abstract void close() throws SQLException; + + @Override + public boolean wasNull() throws SQLException { + return false; + } + + @Override + public abstract String getString(int columnIndex) throws SQLException; + + @Override + public abstract boolean getBoolean(int columnIndex) throws SQLException; + + @Override + public abstract byte getByte(int columnIndex) throws SQLException; + + @Override + public abstract short getShort(int columnIndex) throws SQLException; + + @Override + public abstract int getInt(int columnIndex) throws SQLException; + + @Override + public abstract long getLong(int columnIndex) throws SQLException; + + @Override + public abstract float getFloat(int columnIndex) throws SQLException; + + @Override + public abstract double getDouble(int columnIndex) throws SQLException; + + @Deprecated + @Override + public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { + return getBigDecimal(columnIndex); + } + + @Override + public abstract byte[] getBytes(int columnIndex) throws SQLException; + + @Override + public abstract Date getDate(int columnIndex) throws SQLException; + + @Override + public abstract Time getTime(int columnIndex) throws SQLException; + + @Override + public abstract Timestamp getTimestamp(int columnIndex) throws SQLException; + + @Override + public InputStream getAsciiStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + + } + + @Override + public InputStream getUnicodeStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public InputStream getBinaryStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public String getString(String columnLabel) throws SQLException { + return getString(findColumn(columnLabel)); + } + + @Override + public boolean getBoolean(String columnLabel) throws SQLException { + return getBoolean(findColumn(columnLabel)); + } + + @Override + public byte getByte(String columnLabel) throws SQLException { + return getByte(findColumn(columnLabel)); + } + + @Override + public short getShort(String columnLabel) throws SQLException { + return getShort(findColumn(columnLabel)); + } + + @Override + public int getInt(String columnLabel) throws SQLException { + return getInt(findColumn(columnLabel)); + } + + @Override + public long getLong(String columnLabel) throws SQLException { + return getLong(findColumn(columnLabel)); + } + + @Override + public float getFloat(String columnLabel) throws SQLException { + return getFloat(findColumn(columnLabel)); + } + + @Override + public double getDouble(String columnLabel) throws SQLException { + return getDouble(findColumn(columnLabel)); + } + + @Deprecated + @Override + public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { + return getBigDecimal(findColumn(columnLabel), scale); + } + + @Override + public byte[] getBytes(String columnLabel) throws SQLException { + return getBytes(findColumn(columnLabel)); + } + + @Override + public Date getDate(String columnLabel) throws SQLException { + return getDate(findColumn(columnLabel)); + } + + @Override + public Time getTime(String columnLabel) throws SQLException { + return getTime(findColumn(columnLabel)); + } + + @Override + public Timestamp getTimestamp(String columnLabel) throws SQLException { + return getTimestamp(findColumn(columnLabel)); + } + + @Override + public InputStream getAsciiStream(String columnLabel) throws SQLException { + return getAsciiStream(findColumn(columnLabel)); + } + + @Override + public InputStream getUnicodeStream(String columnLabel) throws SQLException { + return getUnicodeStream(findColumn(columnLabel)); + } + + @Override + public InputStream getBinaryStream(String columnLabel) throws SQLException { + return getBinaryStream(findColumn(columnLabel)); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + return null; + } + + @Override + public void clearWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + } + + @Override + public String getCursorName() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract ResultSetMetaData getMetaData() throws SQLException; + + @Override + public abstract Object getObject(int columnIndex) throws SQLException; + + @Override + public Object getObject(String columnLabel) throws SQLException { + return getObject(findColumn(columnLabel)); + } + + @Override + public abstract int findColumn(String columnLabel) throws SQLException; + + @Override + public Reader getCharacterStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Reader getCharacterStream(String columnLabel) throws SQLException { + return getCharacterStream(findColumn(columnLabel)); + } + + @Override + public abstract BigDecimal getBigDecimal(int columnIndex) throws SQLException; + + @Override + public BigDecimal getBigDecimal(String columnLabel) throws SQLException { + return getBigDecimal(findColumn(columnLabel)); + } + + @Override + public abstract boolean isBeforeFirst() throws SQLException; + + @Override + public abstract boolean isAfterLast() throws SQLException; + + @Override + public abstract boolean isFirst() throws SQLException; + + @Override + public abstract boolean isLast() throws SQLException; + + @Override + public abstract void beforeFirst() throws SQLException; + + @Override + public abstract void afterLast() throws SQLException; + + @Override + public abstract boolean first() throws SQLException; + + @Override + public abstract boolean last() throws SQLException; + + @Override + public abstract int getRow() throws SQLException; + + @Override + public abstract boolean absolute(int row) throws SQLException; + + @Override + public abstract boolean relative(int rows) throws SQLException; + + @Override + public abstract boolean previous() throws SQLException; + + @Override + public void setFetchDirection(int direction) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + //nothing to do + } + + @Override + public int getFetchDirection() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + return ResultSet.FETCH_FORWARD; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (rows < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + //nothing to do + this.fetchSize = rows; + } + + @Override + public int getFetchSize() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + return this.fetchSize; + } + + @Override + public int getType() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + return ResultSet.TYPE_FORWARD_ONLY; + } + + @Override + public int getConcurrency() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + return ResultSet.CONCUR_READ_ONLY; + } + + @Override + public boolean rowUpdated() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean rowInserted() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean rowDeleted() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNull(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBoolean(int columnIndex, boolean x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateByte(int columnIndex, byte x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateShort(int columnIndex, short x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateInt(int columnIndex, int x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateLong(int columnIndex, long x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateFloat(int columnIndex, float x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateDouble(int columnIndex, double x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateString(int columnIndex, String x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBytes(int columnIndex, byte[] x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateDate(int columnIndex, Date x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateTime(int columnIndex, Time x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateObject(int columnIndex, Object x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNull(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBoolean(String columnLabel, boolean x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateByte(String columnLabel, byte x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateShort(String columnLabel, short x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateInt(String columnLabel, int x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateLong(String columnLabel, long x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateFloat(String columnLabel, float x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateDouble(String columnLabel, double x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateString(String columnLabel, String x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBytes(String columnLabel, byte[] x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateDate(String columnLabel, Date x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateTime(String columnLabel, Time x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateObject(String columnLabel, Object x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void insertRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void deleteRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void refreshRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void cancelRowUpdates() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void moveToInsertRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void moveToCurrentRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract Statement getStatement() throws SQLException; + + @Override + public Object getObject(int columnIndex, Map> map) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Ref getRef(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Blob getBlob(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Clob getClob(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Array getArray(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Object getObject(String columnLabel, Map> map) throws SQLException { + return getObject(findColumn(columnLabel), map); + } + + @Override + public Ref getRef(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Blob getBlob(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Clob getClob(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Array getArray(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Date getDate(int columnIndex, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Date getDate(String columnLabel, Calendar cal) throws SQLException { + return getDate(findColumn(columnLabel), cal); + } + + @Override + public Time getTime(int columnIndex, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Time getTime(String columnLabel, Calendar cal) throws SQLException { + return getTime(findColumn(columnLabel), cal); + } + + @Override + public abstract Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException; + + @Override + public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { + return getTimestamp(findColumn(columnLabel), cal); + } + + @Override + public URL getURL(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public URL getURL(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRef(int columnIndex, Ref x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRef(String columnLabel, Ref x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(int columnIndex, Blob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(String columnLabel, Blob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(int columnIndex, Clob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(String columnLabel, Clob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateArray(int columnIndex, Array x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateArray(String columnLabel, Array x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public RowId getRowId(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public RowId getRowId(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRowId(int columnIndex, RowId x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateRowId(String columnLabel, RowId x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int getHoldability() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public abstract boolean isClosed() throws SQLException; + + @Override + public void updateNString(int columnIndex, String nString) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNString(String columnLabel, String nString) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(int columnIndex, NClob nClob) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(String columnLabel, NClob nClob) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public NClob getNClob(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public NClob getNClob(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public SQLXML getSQLXML(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public SQLXML getSQLXML(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract String getNString(int columnIndex) throws SQLException; + + @Override + public String getNString(String columnLabel) throws SQLException { + return getNString(findColumn(columnLabel)); + } + + @Override + public Reader getNCharacterStream(int columnIndex) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public Reader getNCharacterStream(String columnLabel) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(int columnIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateClob(String columnLabel, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(int columnIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void updateNClob(String columnLabel, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public T getObject(int columnIndex, Class type) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public T getObject(String columnLabel, Class type) throws SQLException { + return getObject(findColumn(columnLabel), type); + } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java new file mode 100644 index 0000000000000000000000000000000000000000..9dc559339a4ef88b8423ffcd621c7498437dac5c --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java @@ -0,0 +1,328 @@ +package com.taosdata.jdbc; + +import java.sql.*; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractStatement extends WrapperImpl implements Statement { + + protected List batchedArgs; + private int fetchSize; + + + + @Override + public abstract ResultSet executeQuery(String sql) throws SQLException; + + @Override + public abstract int executeUpdate(String sql) throws SQLException; + + @Override + public abstract void close() throws SQLException; + + @Override + public int getMaxFieldSize() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return TSDBConstants.maxFieldSize; + } + + @Override + public void setMaxFieldSize(int max) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (max < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + // nothing to do + } + + @Override + public int getMaxRows() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return 0; + } + + @Override + public void setMaxRows(int max) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (max < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + // do nothing + } + + @Override + public void setEscapeProcessing(boolean enable) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // do nothing + } + + @Override + public int getQueryTimeout() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return 0; + } + + @Override + public void setQueryTimeout(int seconds) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (seconds < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + @Override + public void cancel() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return null; + } + + @Override + public void clearWarnings() throws SQLException { + // nothing to do + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + } + + @Override + public void setCursorName(String name) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public abstract boolean execute(String sql) throws SQLException; + + @Override + public abstract ResultSet getResultSet() throws SQLException; + + @Override + public abstract int getUpdateCount() throws SQLException; + + @Override + public boolean getMoreResults() throws SQLException { + return getMoreResults(CLOSE_CURRENT_RESULT); + } + + @Override + public void setFetchDirection(int direction) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + switch (direction) { + case ResultSet.FETCH_FORWARD: + case ResultSet.FETCH_REVERSE: + case ResultSet.FETCH_UNKNOWN: + break; + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + //nothing to do + } + + @Override + public int getFetchDirection() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.FETCH_FORWARD; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (rows < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + //nothing to do + this.fetchSize = rows; + } + + @Override + public int getFetchSize() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return this.fetchSize; + } + + @Override + public int getResultSetConcurrency() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.CONCUR_READ_ONLY; + } + + @Override + public int getResultSetType() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.TYPE_FORWARD_ONLY; + } + + @Override + public void addBatch(String sql) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + if (batchedArgs == null) { + batchedArgs = new ArrayList<>(); + } + batchedArgs.add(sql); + } + + @Override + public void clearBatch() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (batchedArgs != null) + batchedArgs.clear(); + } + + @Override + public int[] executeBatch() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (batchedArgs == null || batchedArgs.isEmpty()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_BATCH_IS_EMPTY); + + int[] res = new int[batchedArgs.size()]; + for (int i = 0; i < batchedArgs.size(); i++) { + boolean isSelect = execute(batchedArgs.get(i)); + if (isSelect) { + res[i] = SUCCESS_NO_INFO; + } else { + res[i] = getUpdateCount(); + } + } + return res; + } + + @Override + public abstract Connection getConnection() throws SQLException; + + @Override + public boolean getMoreResults(int current) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + switch (current) { + case Statement.CLOSE_CURRENT_RESULT: + return false; + case Statement.KEEP_CURRENT_RESULT: + case Statement.CLOSE_ALL_RESULTS: + break; + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public ResultSet getGeneratedKeys() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int executeUpdate(String sql, String[] columnNames) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean execute(String sql, int[] columnIndexes) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean execute(String sql, String[] columnNames) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int getResultSetHoldability() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + @Override + public abstract boolean isClosed() throws SQLException; + + @Override + public void setPoolable(boolean poolable) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // do nothing + } + + @Override + public boolean isPoolable() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return false; + } + + @Override + public void closeOnCompletion() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // do nothing + } + + @Override + public boolean isCloseOnCompletion() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + return false; + } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/CatalogResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/CatalogResultSet.java deleted file mode 100644 index 3a01e2e09297d6af1405c63200e2ba91b3fa99a2..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/CatalogResultSet.java +++ /dev/null @@ -1,68 +0,0 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; - -import java.sql.ResultSet; -import java.sql.SQLException; - -/* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the - * standard JDBC API contains more or less some adjustments customized for certain - * compatibility needs. - */ -public class CatalogResultSet extends TSDBResultSetWrapper { - - - public CatalogResultSet(ResultSet resultSet) { - super.setOriginalResultSet(resultSet); - } - - @Override - public String getString(int columnIndex) throws SQLException { - if (columnIndex <= 1) { - return super.getString(columnIndex); - } else { - return null; - } - } - - @Override - public boolean getBoolean(int columnIndex) throws SQLException { - if (columnIndex <= 1) { - return super.getBoolean(columnIndex); - } else { - return false; - } - } - - @Override - public byte[] getBytes(int columnIndex) throws SQLException { - if (columnIndex <= 1) { - return super.getBytes(columnIndex); - } else { - return null; - } - } - - @Override - public Object getObject(int columnIndex) throws SQLException { - if (columnIndex <= 1) { - return super.getObject(columnIndex); - } else { - return null; - } - } - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java index 633fdcd5ab7c9f077abbd725c2511bcc2251db44..14e75f0e09e3403e703658fb503019fefbb6156d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/ColumnMetaData.java @@ -16,40 +16,50 @@ package com.taosdata.jdbc; public class ColumnMetaData { - private int colType = 0; - private String colName = null; - private int colSize = -1; - private int colIndex = 0; - - public int getColSize() { - return colSize; - } - - public void setColSize(int colSize) { - this.colSize = colSize; - } - - public int getColType() { - return colType; - } - - public void setColType(int colType) { - this.colType = colType; - } - - public String getColName() { - return colName; - } - - public void setColName(String colName) { - this.colName = colName; - } - - public int getColIndex() { - return colIndex; - } - - public void setColIndex(int colIndex) { - this.colIndex = colIndex; - } + private int colType = 0; + private String colName = null; + private int colSize = -1; + private int colIndex = 0; + + public int getColSize() { + return colSize; + } + + public void setColSize(int colSize) { + this.colSize = colSize; + } + + public int getColType() { + return colType; + } + + public void setColType(int colType) { + this.colType = colType; + } + + public String getColName() { + return colName; + } + + public void setColName(String colName) { + this.colName = colName; + } + + public int getColIndex() { + return colIndex; + } + + public void setColIndex(int colIndex) { + this.colIndex = colIndex; + } + + @Override + public String toString() { + return "ColumnMetaData{" + + "colType=" + colType + + ", colName='" + colName + '\'' + + ", colSize=" + colSize + + ", colIndex=" + colIndex + + '}'; + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java index f82c064e751c195eb6327580c285a815346c917b..f6a0fcca316cb07d28c71a4e1d51d9405de083ba 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/DatabaseMetaDataResultSet.java @@ -160,12 +160,12 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public Date getDate(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public Time getTime(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -176,17 +176,17 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public InputStream getAsciiStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public InputStream getUnicodeStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public InputStream getBinaryStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -256,22 +256,22 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public InputStream getAsciiStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public InputStream getUnicodeStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public InputStream getBinaryStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -281,7 +281,7 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public String getCursorName() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -308,17 +308,17 @@ public class DatabaseMetaDataResultSet implements ResultSet { return colMetaData.getColIndex() + 1; } } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); } @Override public Reader getCharacterStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public Reader getCharacterStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -353,22 +353,22 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public void beforeFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void afterLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean first() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean last() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -383,17 +383,17 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public boolean absolute(int row) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean relative(int rows) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean previous() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -443,227 +443,227 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public void updateNull(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBoolean(int columnIndex, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateByte(int columnIndex, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateShort(int columnIndex, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateInt(int columnIndex, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateLong(int columnIndex, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateFloat(int columnIndex, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateDouble(int columnIndex, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateString(int columnIndex, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBytes(int columnIndex, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateDate(int columnIndex, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateTime(int columnIndex, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateObject(int columnIndex, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateNull(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBoolean(String columnLabel, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateByte(String columnLabel, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateShort(String columnLabel, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateInt(String columnLabel, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateLong(String columnLabel, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateFloat(String columnLabel, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateDouble(String columnLabel, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateString(String columnLabel, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBytes(String columnLabel, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateDate(String columnLabel, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateTime(String columnLabel, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateObject(String columnLabel, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void insertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void updateRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void deleteRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void refreshRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void cancelRowUpdates() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void moveToInsertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void moveToCurrentRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -673,12 +673,12 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public Object getObject(int columnIndex, Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public Ref getRef(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override @@ -1043,12 +1043,12 @@ public class DatabaseMetaDataResultSet implements ResultSet { @Override public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetColumnsResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetColumnsResultSet.java deleted file mode 100644 index e15415e037948dd7ec757bcdeee03d14a0d588fb..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetColumnsResultSet.java +++ /dev/null @@ -1,51 +0,0 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; - -import java.sql.ResultSet; - -/* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the - * standard JDBC API contains more or less some adjustments customized for certain - * compatibility needs. - */ -public class GetColumnsResultSet extends TSDBResultSetWrapper { - private String catalog; - private String schemaPattern; - private String tableNamePattern; - private String columnNamePattern; - - public GetColumnsResultSet(ResultSet resultSet, String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) { - super.setOriginalResultSet(resultSet); - this.catalog = catalog; - this.schemaPattern = schemaPattern; - this.tableNamePattern = tableNamePattern; - this.columnNamePattern = columnNamePattern; - } - - @Override - public String getString(int columnIndex) { - switch (columnIndex) { - case 1: - return catalog; - case 2: - return null; - case 3: - return tableNamePattern; - default: - return null; - } - } -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetTablesResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetTablesResultSet.java deleted file mode 100644 index e28f6e3c9adf8564437e1214b28b2eb13bdaf8d9..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/GetTablesResultSet.java +++ /dev/null @@ -1,53 +0,0 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; - -import java.sql.ResultSet; -import java.sql.SQLException; - -/* - * TDengine only supports a subset of the standard SQL, thus this implemetation of the - * standard JDBC API contains more or less some adjustments customized for certain - * compatibility needs. - */ -public class GetTablesResultSet extends TSDBResultSetWrapper { - - private String catalog; - private String schemaPattern; - private String tableNamePattern; - private String[] types; - - public GetTablesResultSet(ResultSet resultSet, String catalog, String schemaPattern, String tableNamePattern, String[] types) { - super.setOriginalResultSet(resultSet); - this.catalog = catalog; - this.schemaPattern = schemaPattern; - this.tableNamePattern = tableNamePattern; - this.types = types; - } - - @Override - public String getString(int columnIndex) throws SQLException { - String ret = null; - switch (columnIndex) { - case 3: - return super.getString(1); - case 4: - return "table"; - default: - return null; - } - } - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java deleted file mode 100644 index a0aa7ec584ee351185d10b023e5142d00d60d66b..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/SavedPreparedStatement.java +++ /dev/null @@ -1,453 +0,0 @@ -package com.taosdata.jdbc; - -import com.taosdata.jdbc.bean.TSDBPreparedParam; - -import java.sql.SQLException; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * this class is used to precompile the sql of tdengine insert or import ops - */ -public class SavedPreparedStatement { - - private TSDBPreparedStatement tsdbPreparedStatement; - - /** - * sql param List - */ - private List sqlParamList; - - /** - * init param according the sql - */ - private TSDBPreparedParam initPreparedParam; - - /** - * is table name dynamic in the prepared sql - */ - private boolean isTableNameDynamic; - - /** - * insert or import sql template pattern, the template are the following: - *

    - * insert/import into tableName [(field1, field2, ...)] [using stables tags(?, ?, ...) ] values(?, ?, ...) (?, ?, ...) - *

    - * we split it to three part: - * 1. prefix, insert/import - * 2. middle, tableName [(field1, field2, ...)] [using stables tags(?, ?, ...) ] - * 3. valueList, the content after values, for example (?, ?, ...) (?, ?, ...) - */ - private Pattern sqlPattern = Pattern.compile("(?s)(?i)^\\s*(INSERT|IMPORT)\\s+INTO\\s+((?\\S+)\\s*(\\(.*\\))?\\s+(USING\\s+(?\\S+)\\s+TAGS\\s*\\((?.+)\\))?)\\s*VALUES\\s*(?\\(.*\\)).*"); - - /** - * the raw sql template - */ - private String sql; - - /** - * the prefix part of sql - */ - private String prefix; - - /** - * the middle part of sql - */ - private String middle; - - private int middleParamSize; - - /** - * the valueList part of sql - */ - private String valueList; - - private int valueListSize; - - /** - * default param value - */ - private static final String DEFAULT_VALUE = "NULL"; - - private static final String PLACEHOLDER = "?"; - - private String tableName; - - /** - * is the parameter add to batch list - */ - private boolean isAddBatch; - - public SavedPreparedStatement(String sql, TSDBPreparedStatement tsdbPreparedStatement) throws SQLException { - this.sql = sql; - this.tsdbPreparedStatement = tsdbPreparedStatement; - this.sqlParamList = new ArrayList<>(); - - parsePreparedParam(this.sql); - } - - /** - * parse the init param according the sql param - * - * @param sql - */ - private void parsePreparedParam(String sql) throws SQLException { - - Matcher matcher = sqlPattern.matcher(sql); - - if (matcher.find()) { - - tableName = matcher.group("tablename"); - - if (tableName != null && PLACEHOLDER.equals(tableName)) { - // the table name is dynamic - this.isTableNameDynamic = true; - } - - prefix = matcher.group(1); - middle = matcher.group(2); - valueList = matcher.group("valueList"); - - if (middle != null && !"".equals(middle)) { - middleParamSize = parsePlaceholder(middle); - } - - if (valueList != null && !"".equals(valueList)) { - valueListSize = parsePlaceholder(valueList); - } - - initPreparedParam = initDefaultParam(tableName, middleParamSize, valueListSize); - - } else { - // not match - throw new SQLException(TSDBConstants.WrapErrMsg("the sql is not complete!")); - } - - } - - private TSDBPreparedParam initDefaultParam(String tableName, int middleParamSize, int valueListSize) { - - TSDBPreparedParam tsdbPreparedParam = new TSDBPreparedParam(tableName); - - tsdbPreparedParam.setMiddleParamList(getDefaultParamList(middleParamSize)); - - tsdbPreparedParam.setValueList(getDefaultParamList(valueListSize)); - - return tsdbPreparedParam; - } - - /** - * generate the default param value list - * - * @param paramSize - * @return - */ - private List getDefaultParamList(int paramSize) { - - List paramList = new ArrayList<>(paramSize); - if (paramSize > 0) { - for (int i = 0; i < paramSize; i++) { - paramList.add(i, DEFAULT_VALUE); - } - } - - return paramList; - } - - /** - * calculate the placeholder num - * - * @param value - * @return - */ - private int parsePlaceholder(String value) { - - Pattern pattern = Pattern.compile("[?]"); - - Matcher matcher = pattern.matcher(value); - - int result = 0; - while (matcher.find()) { - result++; - } - return result; - } - - /** - * set current row params - * - * @param parameterIndex the first parameter is 1, the second is 2, ... - * @param x the parameter value - */ - public void setParam(int parameterIndex, Object x) throws SQLException { - - int paramSize = this.middleParamSize + this.valueListSize; - - String errorMsg = String.format("the parameterIndex %s out of the range [1, %s]", parameterIndex, paramSize); - - if (parameterIndex < 1 || parameterIndex > paramSize) { - throw new SQLException(TSDBConstants.WrapErrMsg(errorMsg)); - } - - this.isAddBatch = false; //set isAddBatch to false - - if (x == null) { - x = DEFAULT_VALUE; // set default null string - } - - parameterIndex = parameterIndex - 1; // start from 0 in param list - - if (this.middleParamSize > 0 && parameterIndex >= 0 && parameterIndex < this.middleParamSize) { - - this.initPreparedParam.setMiddleParam(parameterIndex, x); - return; - } - - if (this.valueListSize > 0 && parameterIndex >= this.middleParamSize && parameterIndex < paramSize) { - - this.initPreparedParam.setValueParam(parameterIndex - this.middleParamSize, x); - return; - } - - throw new SQLException(TSDBConstants.WrapErrMsg(errorMsg)); - } - - public void addBatch() { - - addCurrentRowParamToList(); - this.initPreparedParam = initDefaultParam(tableName, middleParamSize, valueListSize); - } - - /** - * add current param to batch list - */ - private void addCurrentRowParamToList() { - - if (initPreparedParam != null && (this.middleParamSize > 0 || this.valueListSize > 0)) { - this.sqlParamList.add(initPreparedParam); // add current param to batch list - } - this.isAddBatch = true; - } - - - /** - * execute the sql with batch sql - * - * @return - * @throws SQLException - */ - public int[] executeBatch() throws SQLException { - - int result = executeBatchInternal(); - - return new int[]{result}; - } - - - public int executeBatchInternal() throws SQLException { - - if (!isAddBatch) { - addCurrentRowParamToList(); // add current param to batch list - } - - //1. generate batch sql - String sql = generateExecuteSql(); - //2. execute batch sql - int result = executeSql(sql); - - //3. clear batch param list - this.sqlParamList.clear(); - - return result; - } - - /** - * generate the batch sql - * - * @return - */ - private String generateExecuteSql() { - - StringBuilder stringBuilder = new StringBuilder(); - - stringBuilder.append(prefix); - stringBuilder.append(" into "); - - if (!isTableNameDynamic) { - // tablename will not need to be replaced - String middleValue = replaceMiddleListParam(middle, sqlParamList); - stringBuilder.append(middleValue); - stringBuilder.append(" values"); - - stringBuilder.append(replaceValueListParam(valueList, sqlParamList)); - - } else { - // need to replace tablename - - if (sqlParamList.size() > 0) { - - TSDBPreparedParam firstPreparedParam = sqlParamList.get(0); - - //replace middle part and value part of first row - String firstRow = replaceMiddleAndValuePart(firstPreparedParam); - stringBuilder.append(firstRow); - - //the first param in the middleParamList is the tableName - String lastTableName = firstPreparedParam.getMiddleParamList().get(0).toString(); - - if (sqlParamList.size() > 1) { - - for (int i = 1; i < sqlParamList.size(); i++) { - TSDBPreparedParam currentParam = sqlParamList.get(i); - String currentTableName = currentParam.getMiddleParamList().get(0).toString(); - if (lastTableName.equalsIgnoreCase(currentTableName)) { - // tablename is same with the last row ,so only need to append the part of value - - String values = replaceTemplateParam(valueList, currentParam.getValueList()); - stringBuilder.append(values); - - } else { - // tablename difference with the last row - //need to replace middle part and value part - String row = replaceMiddleAndValuePart(currentParam); - stringBuilder.append(row); - lastTableName = currentTableName; - } - } - } - - } else { - - stringBuilder.append(middle); - stringBuilder.append(" values"); - stringBuilder.append(valueList); - } - - } - - return stringBuilder.toString(); - } - - /** - * replace the middle and value part - * - * @param tsdbPreparedParam - * @return - */ - private String replaceMiddleAndValuePart(TSDBPreparedParam tsdbPreparedParam) { - - StringBuilder stringBuilder = new StringBuilder(" "); - - String middlePart = replaceTemplateParam(middle, tsdbPreparedParam.getMiddleParamList()); - - stringBuilder.append(middlePart); - stringBuilder.append(" values "); - - String valuePart = replaceTemplateParam(valueList, tsdbPreparedParam.getValueList()); - stringBuilder.append(valuePart); - stringBuilder.append(" "); - - return stringBuilder.toString(); - } - - /** - * replace the placeholder of the middle part of sql template with TSDBPreparedParam list - * - * @param template - * @param sqlParamList - * @return - */ - private String replaceMiddleListParam(String template, List sqlParamList) { - - if (sqlParamList.size() > 0) { - - //becase once the subTableName is static then will be ignore the tag which after the first setTag - return replaceTemplateParam(template, sqlParamList.get(0).getMiddleParamList()); - - } - - return template; - } - - - /** - * replace the placeholder of the template with TSDBPreparedParam list - * - * @param template - * @param sqlParamList - * @return - */ - private String replaceValueListParam(String template, List sqlParamList) { - - StringBuilder stringBuilder = new StringBuilder(); - - if (sqlParamList.size() > 0) { - - for (TSDBPreparedParam tsdbPreparedParam : sqlParamList) { - - String tmp = replaceTemplateParam(template, tsdbPreparedParam.getValueList()); - - stringBuilder.append(tmp); - } - - } else { - stringBuilder.append(template); - } - - return stringBuilder.toString(); - } - - /** - * replace the placeholder of the template with paramList - * - * @param template - * @param paramList - * @return - */ - private String replaceTemplateParam(String template, List paramList) { - - if (paramList.size() > 0) { - - String tmp = template; - - for (int i = 0; i < paramList.size(); ++i) { - - String paraStr = getParamString(paramList.get(i)); - - tmp = tmp.replaceFirst("[" + PLACEHOLDER + "]", paraStr); - - } - - return tmp; - - } else { - return template; - } - - } - - /** - * get the string of param object - * - * @param paramObj - * @return - */ - private String getParamString(Object paramObj) { - - String paraStr = paramObj.toString(); - if (paramObj instanceof Timestamp || (paramObj instanceof String && !DEFAULT_VALUE.equalsIgnoreCase(paraStr))) { - paraStr = "'" + paraStr + "'"; - } - return paraStr; - } - - - private int executeSql(String sql) throws SQLException { - - return tsdbPreparedStatement.executeUpdate(sql); - } - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 94abe3965507170a4b31e17ebb431ddcb4fa11f8..5b4615485dd21b7d5e41e513f48116edf53594d0 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -14,65 +14,38 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.sql.Array; -import java.sql.Blob; -import java.sql.CallableStatement; -import java.sql.Clob; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.NClob; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLClientInfoException; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Savepoint; -import java.sql.Statement; -import java.sql.Struct; -import java.util.*; -import java.util.concurrent.Executor; +import java.sql.*; +import java.util.Properties; -public class TSDBConnection implements Connection { - protected Properties props = null; +public class TSDBConnection extends AbstractConnection { - private TSDBJNIConnector connector = null; + private TSDBJNIConnector connector; + private TSDBDatabaseMetaData databaseMetaData; + private boolean batchFetch; - private String catalog = null; - - private TSDBDatabaseMetaData dbMetaData = null; - - private Properties clientInfoProps = new Properties(); - - private int timeoutMilliseconds = 0; - - private boolean batchFetch = false; + public Boolean getBatchFetch() { + return this.batchFetch; + } public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { - this.dbMetaData = meta; + this.databaseMetaData = meta; connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), - info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), + info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), info.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD)); - + String batchLoad = info.getProperty(TSDBDriver.PROPERTY_KEY_BATCH_LOAD); if (batchLoad != null) { - this.batchFetch = Boolean.parseBoolean(batchLoad); + this.batchFetch = Boolean.parseBoolean(batchLoad); } } private void connect(String host, int port, String dbName, String user, String password) throws SQLException { this.connector = new TSDBJNIConnector(); this.connector.connect(host, port, dbName, user, password); - - try { - this.setCatalog(dbName); - } catch (SQLException e) { - e.printStackTrace(); - } - - this.dbMetaData.setConnection(this); + this.catalog = dbName; + this.databaseMetaData.setConnection(this); } public TSDBJNIConnector getConnection() { @@ -80,304 +53,47 @@ public class TSDBConnection implements Connection { } public Statement createStatement() throws SQLException { - if (!this.connector.isClosed()) { - TSDBStatement statement = new TSDBStatement(this, this.connector); - statement.setConnection(this); - return statement; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + + return new TSDBStatement(this, this.connector); } public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { - if (this.connector.isClosed()) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } long id = this.connector.subscribe(topic, sql, restart, 0); if (id == 0) { - throw new SQLException(TSDBConstants.WrapErrMsg("failed to create subscription")); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED); } - return new TSDBSubscribe(this.connector, id); } public PreparedStatement prepareStatement(String sql) throws SQLException { - if (!this.connector.isClosed()) { - return new TSDBPreparedStatement(this, this.connector, sql); - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - } - - public CallableStatement prepareCall(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String nativeSQL(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setAutoCommit(boolean autoCommit) throws SQLException { - } - - public boolean getAutoCommit() throws SQLException { - return true; - } - - public void commit() throws SQLException { - } - - public void rollback() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return new TSDBPreparedStatement(this, this.connector, sql); } public void close() throws SQLException { - if (this.connector != null && !this.connector.isClosed()) { - this.connector.closeConnection(); - } else { - throw new SQLException(TSDBConstants.WrapErrMsg("connection is already closed!")); - } + if (isClosed) + return; + this.connector.closeConnection(); + this.isClosed = true; } public boolean isClosed() throws SQLException { - return this.connector.isClosed(); + return this.connector != null && this.connector.isClosed(); } - /** - * A connection's database is able to provide information describing its tables, - * its supported SQL grammar, its stored procedures, the capabilities of this - * connection, etc. This information is made available through a - * DatabaseMetaData object. - * - * @return a DatabaseMetaData object for this connection - * @throws SQLException if a database access error occurs - */ public DatabaseMetaData getMetaData() throws SQLException { - return this.dbMetaData; - } - - /** - * This readOnly option is not supported by TDengine. However, the method is intentionally left blank here to - * support HikariCP connection. - * - * @param readOnly - * @throws SQLException - */ - public void setReadOnly(boolean readOnly) throws SQLException { - } - - public boolean isReadOnly() throws SQLException { - return true; - } - - public void setCatalog(String catalog) throws SQLException { - this.catalog = catalog; - } - - public String getCatalog() throws SQLException { - return this.catalog; - } - - /** - * The transaction isolation level option is not supported by TDengine. - * This method is intentionally left empty to support HikariCP connection. - * - * @param level - * @throws SQLException - */ - public void setTransactionIsolation(int level) throws SQLException { - } - - /** - * The transaction isolation level option is not supported by TDengine. - * - * @return - * @throws SQLException - */ - public int getTransactionIsolation() throws SQLException { - return Connection.TRANSACTION_NONE; - } - - public SQLWarning getWarnings() throws SQLException { - //todo: implement getWarnings according to the warning messages returned from TDengine - return null; -// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - // left blank to support HikariCP connection - //todo: implement clearWarnings according to the warning messages returned from TDengine - } - - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) - throws SQLException { - // This method is implemented in the current way to support Spark - if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); + if (isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } - - if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - return this.prepareStatement(sql); - } - - public Boolean getBatchFetch() { - return this.batchFetch; - } - - public void setBatchFetch(Boolean batchFetch) { - this.batchFetch = batchFetch; - } - - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Map> getTypeMap() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setTypeMap(Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setHoldability(int holdability) throws SQLException { - // intentionally left empty to support druid connection pool. + return this.databaseMetaData; } - /** - * the transaction is not supported by TDengine, so the opened ResultSet Objects will remain open - * - * @return - * @throws SQLException - */ - public int getHoldability() throws SQLException { - //intentionally left empty to support HikariCP connection. - return ResultSet.HOLD_CURSORS_OVER_COMMIT; - } - - public Savepoint setSavepoint() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Savepoint setSavepoint(String name) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void rollback(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void releaseSavepoint(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, - int resultSetHoldability) throws SQLException { - return this.prepareStatement(sql, resultSetType, resultSetConcurrency); - } - - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, - int resultSetHoldability) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Clob createClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Blob createBlob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob createNClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML createSQLXML() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isValid(int timeout) throws SQLException { - return !this.isClosed(); - } - - public void setClientInfo(String name, String value) throws SQLClientInfoException { - clientInfoProps.setProperty(name, value); - } - - public void setClientInfo(Properties properties) throws SQLClientInfoException { - for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { - String name = (String) enumer.nextElement(); - clientInfoProps.put(name, properties.getProperty(name)); - } - } - - public String getClientInfo(String name) throws SQLException { - return clientInfoProps.getProperty(name); - } - - public Properties getClientInfo() throws SQLException { - return clientInfoProps; - } - - public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setSchema(String schema) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getSchema() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void abort(Executor executor) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { - this.timeoutMilliseconds = milliseconds; - } - - public int getNetworkTimeout() throws SQLException { - return this.timeoutMilliseconds; - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } -} +} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 4f4911aad9c138eb13fffdd698b794a03222160f..37073e243fef99176c6d3d16d87d7cdaabb0e1b4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -14,18 +14,13 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.util.HashMap; -import java.util.Map; +import java.sql.SQLException; +import java.sql.Types; public abstract class TSDBConstants { - public static final String DEFAULT_PORT = "6200"; - public static final String UNSUPPORT_METHOD_EXCEPTIONZ_MSG = "this operation is NOT supported currently!"; - public static final String INVALID_VARIABLES = "invalid variables"; - public static Map DATATYPE_MAP = null; - public static final long JNI_NULL_POINTER = 0L; - + // JNI_ERROR_NUMBER public static final int JNI_SUCCESS = 0; public static final int JNI_TDENGINE_ERROR = -1; public static final int JNI_CONNECTION_NULL = -2; @@ -33,8 +28,8 @@ public abstract class TSDBConstants { public static final int JNI_NUM_OF_FIELDS_0 = -4; public static final int JNI_SQL_NULL = -5; public static final int JNI_FETCH_END = -6; - - public static final int TSDB_DATA_TYPE_NULL = 0; + public static final int JNI_OUT_OF_MEMORY = -7; + // TSDB Data Types public static final int TSDB_DATA_TYPE_BOOL = 1; public static final int TSDB_DATA_TYPE_TINYINT = 2; public static final int TSDB_DATA_TYPE_SMALLINT = 3; @@ -45,45 +40,134 @@ public abstract class TSDBConstants { public static final int TSDB_DATA_TYPE_BINARY = 8; public static final int TSDB_DATA_TYPE_TIMESTAMP = 9; public static final int TSDB_DATA_TYPE_NCHAR = 10; - - // nchar field's max length + /* + 系统增加新的无符号数据类型,分别是: + unsigned tinyint, 数值范围:0-254, NULL 为255 + unsigned smallint,数值范围: 0-65534, NULL 为65535 + unsigned int,数值范围:0-4294967294,NULL 为4294967295u + unsigned bigint,数值范围:0-18446744073709551614u,NULL 为18446744073709551615u。 + example: + create table tb(ts timestamp, a tinyint unsigned, b smallint unsigned, c int unsigned, d bigint unsigned); + */ + public static final int TSDB_DATA_TYPE_UTINYINT = 11; //unsigned tinyint + public static final int TSDB_DATA_TYPE_USMALLINT = 12; //unsigned smallint + public static final int TSDB_DATA_TYPE_UINT = 13; //unsigned int + public static final int TSDB_DATA_TYPE_UBIGINT = 14; //unsigned bigint + // nchar column max length public static final int maxFieldSize = 16 * 1024; - public static String WrapErrMsg(String msg) { - return "TDengine Error: " + msg; + public static int taosType2JdbcType(int taosType) throws SQLException { + switch (taosType) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return Types.BOOLEAN; + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: + return Types.TINYINT; + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return Types.SMALLINT; + case TSDBConstants.TSDB_DATA_TYPE_UINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: + return Types.INTEGER; + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return Types.BIGINT; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return Types.FLOAT; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return Types.DOUBLE; + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + return Types.BINARY; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + return Types.TIMESTAMP; + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + return Types.NCHAR; + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE); } - public static String FixErrMsg(int code) { - switch (code) { - case JNI_TDENGINE_ERROR: - return WrapErrMsg("internal error of database!"); - case JNI_CONNECTION_NULL: - return WrapErrMsg("invalid tdengine connection!"); - case JNI_RESULT_SET_NULL: - return WrapErrMsg("invalid resultset pointer!"); - case JNI_NUM_OF_FIELDS_0: - return WrapErrMsg("invalid num of fields!"); - case JNI_SQL_NULL: - return WrapErrMsg("can't execute empty sql!"); - case JNI_FETCH_END: - return WrapErrMsg("fetch to the end of resultset"); + public static String taosType2JdbcTypeName(int taosType) throws SQLException { + switch (taosType){ + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return "BOOL"; + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return "TINYINT"; + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return "SMALLINT"; + case TSDBConstants.TSDB_DATA_TYPE_UINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: + return "INT"; + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return "BIGINT"; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return "FLOAT"; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return "DOUBLE"; + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + return "BINARY"; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + return "TIMESTAMP"; + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + return "NCHAR"; default: - break; + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE); } - return WrapErrMsg("unkown error!"); } - static { - DATATYPE_MAP = new HashMap(); - DATATYPE_MAP.put(1, "BOOL"); - DATATYPE_MAP.put(2, "TINYINT"); - DATATYPE_MAP.put(3, "SMALLINT"); - DATATYPE_MAP.put(4, "INT"); - DATATYPE_MAP.put(5, "BIGINT"); - DATATYPE_MAP.put(6, "FLOAT"); - DATATYPE_MAP.put(7, "DOUBLE"); - DATATYPE_MAP.put(8, "BINARY"); - DATATYPE_MAP.put(9, "TIMESTAMP"); - DATATYPE_MAP.put(10, "NCHAR"); + public static int jdbcType2TaosType(int jdbcType) throws SQLException { + switch (jdbcType){ + case Types.BOOLEAN: + return TSDBConstants.TSDB_DATA_TYPE_BOOL; + case Types.TINYINT: + return TSDBConstants.TSDB_DATA_TYPE_TINYINT; + case Types.SMALLINT: + return TSDBConstants.TSDB_DATA_TYPE_SMALLINT; + case Types.INTEGER: + return TSDBConstants.TSDB_DATA_TYPE_INT; + case Types.BIGINT: + return TSDBConstants.TSDB_DATA_TYPE_BIGINT; + case Types.FLOAT: + return TSDBConstants.TSDB_DATA_TYPE_FLOAT; + case Types.DOUBLE: + return TSDBConstants.TSDB_DATA_TYPE_DOUBLE; + case Types.BINARY: + return TSDBConstants.TSDB_DATA_TYPE_BINARY; + case Types.TIMESTAMP: + return TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP; + case Types.NCHAR: + return TSDBConstants.TSDB_DATA_TYPE_NCHAR; + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); } + + public static String jdbcType2TaosTypeName(int jdbcType) throws SQLException { + switch (jdbcType){ + case Types.BOOLEAN: + return "BOOL"; + case Types.TINYINT: + return "TINYINT"; + case Types.SMALLINT: + return "SMALLINT"; + case Types.INTEGER: + return "INT"; + case Types.BIGINT: + return "BIGINT"; + case Types.FLOAT: + return "FLOAT"; + case Types.DOUBLE: + return "DOUBLE"; + case Types.BINARY: + return "BINARY"; + case Types.TIMESTAMP: + return "TIMESTAMP"; + case Types.NCHAR: + return "NCHAR"; + default: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); + } + } + } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java index f4dee67adf03a3474f582dbe662a8c5e988e3fab..8b7ede148e89cce0d8db22e62627bd1e1c49f9bb 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDatabaseMetaData.java @@ -14,41 +14,27 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.sql.*; -import java.util.ArrayList; -import java.util.List; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; -public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData { +public class TSDBDatabaseMetaData extends AbstractDatabaseMetaData { - private String dbProductName = null; - private String url = null; - private String userName = null; - private Connection conn = null; + private String url; + private String userName; + private Connection conn; - public TSDBDatabaseMetaData(String dbProductName, String url, String userName) { - this.dbProductName = dbProductName; + public TSDBDatabaseMetaData(String url, String userName) { this.url = url; this.userName = userName; } - public void setConnection(Connection conn) { - this.conn = conn; - } - - public T unwrap(Class iface) throws SQLException { - return null; - } - - public boolean isWrapperFor(Class iface) throws SQLException { - return false; - } - - public boolean allProceduresAreCallable() throws SQLException { - return false; + public Connection getConnection() throws SQLException { + return this.conn; } - public boolean allTablesAreSelectable() throws SQLException { - return false; + public void setConnection(Connection conn) { + this.conn = conn; } public String getURL() throws SQLException { @@ -59,906 +45,51 @@ public class TSDBDatabaseMetaData implements java.sql.DatabaseMetaData { return this.userName; } - public boolean isReadOnly() throws SQLException { - return false; - } - - public boolean nullsAreSortedHigh() throws SQLException { - return false; - } - - public boolean nullsAreSortedLow() throws SQLException { - return !nullsAreSortedHigh(); - } - - public boolean nullsAreSortedAtStart() throws SQLException { - return true; - } - - public boolean nullsAreSortedAtEnd() throws SQLException { - return !nullsAreSortedAtStart(); - } - - public String getDatabaseProductName() throws SQLException { - return this.dbProductName; - } - - public String getDatabaseProductVersion() throws SQLException { - return "1.5.1"; - } - public String getDriverName() throws SQLException { return TSDBDriver.class.getName(); } - public String getDriverVersion() throws SQLException { - return "1.0.0"; - } - - public int getDriverMajorVersion() { - return 2; - } - - public int getDriverMinorVersion() { - return 0; - } - - public boolean usesLocalFiles() throws SQLException { - return false; - } - - public boolean usesLocalFilePerTable() throws SQLException { - return false; - } - - public boolean supportsMixedCaseIdentifiers() throws SQLException { - return false; - } - - public boolean storesUpperCaseIdentifiers() throws SQLException { - return false; - } - - public boolean storesLowerCaseIdentifiers() throws SQLException { - return false; - } - - public boolean storesMixedCaseIdentifiers() throws SQLException { - return false; - } - - public boolean supportsMixedCaseQuotedIdentifiers() throws SQLException { - return false; - } - - public boolean storesUpperCaseQuotedIdentifiers() throws SQLException { - return false; - } - - public boolean storesLowerCaseQuotedIdentifiers() throws SQLException { - return false; - } - - public boolean storesMixedCaseQuotedIdentifiers() throws SQLException { - return false; - } - - public String getIdentifierQuoteString() throws SQLException { - return " "; - } - - public String getSQLKeywords() throws SQLException { - return null; - } - - public String getNumericFunctions() throws SQLException { - return null; - } - - public String getStringFunctions() throws SQLException { - return null; - } - - public String getSystemFunctions() throws SQLException { - return null; - } - - public String getTimeDateFunctions() throws SQLException { - return null; - } - - public String getSearchStringEscape() throws SQLException { - return null; - } - - public String getExtraNameCharacters() throws SQLException { - return null; - } - - public boolean supportsAlterTableWithAddColumn() throws SQLException { - return true; - } - - public boolean supportsAlterTableWithDropColumn() throws SQLException { - return true; - } - - public boolean supportsColumnAliasing() throws SQLException { - return true; - } - - public boolean nullPlusNonNullIsNull() throws SQLException { - return false; - } - - public boolean supportsConvert() throws SQLException { - return false; - } - - public boolean supportsConvert(int fromType, int toType) throws SQLException { - return false; - } - - public boolean supportsTableCorrelationNames() throws SQLException { - return false; - } - - public boolean supportsDifferentTableCorrelationNames() throws SQLException { - return false; - } - - public boolean supportsExpressionsInOrderBy() throws SQLException { - return false; - } - - public boolean supportsOrderByUnrelated() throws SQLException { - return false; - } - - public boolean supportsGroupBy() throws SQLException { - return false; - } - - public boolean supportsGroupByUnrelated() throws SQLException { - return false; - } - - public boolean supportsGroupByBeyondSelect() throws SQLException { - return false; - } - - public boolean supportsLikeEscapeClause() throws SQLException { - return false; - } - - public boolean supportsMultipleResultSets() throws SQLException { - return false; - } - - public boolean supportsMultipleTransactions() throws SQLException { - return false; - } - - public boolean supportsNonNullableColumns() throws SQLException { - return false; - } - - public boolean supportsMinimumSQLGrammar() throws SQLException { - return false; - } - - public boolean supportsCoreSQLGrammar() throws SQLException { - return false; - } - - public boolean supportsExtendedSQLGrammar() throws SQLException { - return false; - } - - public boolean supportsANSI92EntryLevelSQL() throws SQLException { - return false; - } - - public boolean supportsANSI92IntermediateSQL() throws SQLException { - return false; - } - - public boolean supportsANSI92FullSQL() throws SQLException { - return false; - } - - public boolean supportsIntegrityEnhancementFacility() throws SQLException { - return false; - } - - public boolean supportsOuterJoins() throws SQLException { - return false; - } - - public boolean supportsFullOuterJoins() throws SQLException { - return false; - } - - public boolean supportsLimitedOuterJoins() throws SQLException { - return false; - } - - public String getSchemaTerm() throws SQLException { - return null; - } - - public String getProcedureTerm() throws SQLException { - return null; - } - - public String getCatalogTerm() throws SQLException { - return "database"; - } - - public boolean isCatalogAtStart() throws SQLException { - return true; - } - - public String getCatalogSeparator() throws SQLException { - return "."; - } - - public boolean supportsSchemasInDataManipulation() throws SQLException { - return false; - } - - public boolean supportsSchemasInProcedureCalls() throws SQLException { - return false; - } - - public boolean supportsSchemasInTableDefinitions() throws SQLException { - return false; - } - - public boolean supportsSchemasInIndexDefinitions() throws SQLException { - return false; - } - - public boolean supportsSchemasInPrivilegeDefinitions() throws SQLException { - return false; - } - - public boolean supportsCatalogsInDataManipulation() throws SQLException { - return true; - } - - public boolean supportsCatalogsInProcedureCalls() throws SQLException { - return false; - } - - public boolean supportsCatalogsInTableDefinitions() throws SQLException { - return false; - } - - public boolean supportsCatalogsInIndexDefinitions() throws SQLException { - return false; - } - - public boolean supportsCatalogsInPrivilegeDefinitions() throws SQLException { - return false; - } - - public boolean supportsPositionedDelete() throws SQLException { - return false; - } - - public boolean supportsPositionedUpdate() throws SQLException { - return false; - } - - public boolean supportsSelectForUpdate() throws SQLException { - return false; - } - - public boolean supportsStoredProcedures() throws SQLException { - return false; - } - - public boolean supportsSubqueriesInComparisons() throws SQLException { - return false; - } - - public boolean supportsSubqueriesInExists() throws SQLException { - return false; - } - - public boolean supportsSubqueriesInIns() throws SQLException { - return false; - } - - public boolean supportsSubqueriesInQuantifieds() throws SQLException { - return false; - } - - public boolean supportsCorrelatedSubqueries() throws SQLException { - return false; - } - - public boolean supportsUnion() throws SQLException { - return false; - } - - public boolean supportsUnionAll() throws SQLException { - return false; - } - - public boolean supportsOpenCursorsAcrossCommit() throws SQLException { - return false; - } - - public boolean supportsOpenCursorsAcrossRollback() throws SQLException { - return false; - } - - public boolean supportsOpenStatementsAcrossCommit() throws SQLException { - return false; - } - - public boolean supportsOpenStatementsAcrossRollback() throws SQLException { - return false; - } - - public int getMaxBinaryLiteralLength() throws SQLException { - return 0; - } - - public int getMaxCharLiteralLength() throws SQLException { - return 0; - } - - public int getMaxColumnNameLength() throws SQLException { - return 0; - } - - public int getMaxColumnsInGroupBy() throws SQLException { - return 0; - } - - public int getMaxColumnsInIndex() throws SQLException { - return 0; - } - - public int getMaxColumnsInOrderBy() throws SQLException { - return 0; - } - - public int getMaxColumnsInSelect() throws SQLException { - return 0; - } - - public int getMaxColumnsInTable() throws SQLException { - return 0; - } - - public int getMaxConnections() throws SQLException { - return 0; - } - - public int getMaxCursorNameLength() throws SQLException { - return 0; - } - - public int getMaxIndexLength() throws SQLException { - return 0; - } - - public int getMaxSchemaNameLength() throws SQLException { - return 0; - } - - public int getMaxProcedureNameLength() throws SQLException { - return 0; - } - - public int getMaxCatalogNameLength() throws SQLException { - return 0; - } - - public int getMaxRowSize() throws SQLException { - return 0; - } - - public boolean doesMaxRowSizeIncludeBlobs() throws SQLException { - return false; - } - - public int getMaxStatementLength() throws SQLException { - return 0; - } - - public int getMaxStatements() throws SQLException { - return 0; - } - - public int getMaxTableNameLength() throws SQLException { - return 0; - } - - public int getMaxTablesInSelect() throws SQLException { - return 0; - } - - public int getMaxUserNameLength() throws SQLException { - return 0; - } - - public int getDefaultTransactionIsolation() throws SQLException { - return 0; - } - - public boolean supportsTransactions() throws SQLException { - return false; - } - - public boolean supportsTransactionIsolationLevel(int level) throws SQLException { - return false; - } - - public boolean supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { - return false; - } - - public boolean supportsDataManipulationTransactionsOnly() throws SQLException { - return false; - } - - public boolean dataDefinitionCausesTransactionCommit() throws SQLException { - return false; - } - - public boolean dataDefinitionIgnoredInTransactions() throws SQLException { - return false; - } - - public ResultSet getProcedures(String catalog, String schemaPattern, String procedureNamePattern) - throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSet getProcedureColumns(String catalog, String schemaPattern, String procedureNamePattern, - String columnNamePattern) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) - throws SQLException { - Statement stmt = null; - if (null != conn && !conn.isClosed()) { - stmt = conn.createStatement(); - if (catalog == null || catalog.length() < 1) { - catalog = conn.getCatalog(); - } - stmt.executeUpdate("use " + catalog); - ResultSet resultSet0 = stmt.executeQuery("show tables"); - GetTablesResultSet getTablesResultSet = new GetTablesResultSet(resultSet0, catalog, schemaPattern, tableNamePattern, types); - return getTablesResultSet; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + /** + * @Param catalog : database名称,"" 表示不属于任何database的table,null表示不使用database来缩小范围 + * @Param schemaPattern : schema名称,""表示 + * @Param tableNamePattern : 表名满足tableNamePattern的表, null表示返回所有表 + * @Param types : 表类型,null表示返回所有类型 + */ + public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { + if (conn == null || conn.isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } - } - - public ResultSet getSchemas() throws SQLException { - return getEmptyResultSet(); + return super.getTables(catalog, schemaPattern, tableNamePattern, types, conn); } public ResultSet getCatalogs() throws SQLException { - - if (conn != null && !conn.isClosed()) { - Statement stmt = conn.createStatement(); - ResultSet resultSet0 = stmt.executeQuery("show databases"); - CatalogResultSet resultSet = new CatalogResultSet(resultSet0); - return resultSet; - } else { - return getEmptyResultSet(); - } + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getCatalogs(conn); } public ResultSet getTableTypes() throws SQLException { - DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); - - // set up ColumnMetaDataList - List columnMetaDataList = new ArrayList(1); - ColumnMetaData colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(0); - colMetaData.setColName("TABLE_TYPE"); - colMetaData.setColSize(10); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - - // set up rowDataList - List rowDataList = new ArrayList(2); - TSDBResultSetRowData rowData = new TSDBResultSetRowData(); - rowData.setString(0, "TABLE"); - rowDataList.add(rowData); - rowData = new TSDBResultSetRowData(); - rowData.setString(0, "STABLE"); - rowDataList.add(rowData); - - resultSet.setColumnMetaDataList(columnMetaDataList); - resultSet.setRowDataList(rowDataList); - return resultSet; - } - - public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) - throws SQLException { - - /** add by zyyang **********/ - Statement stmt = null; - if (null != conn && !conn.isClosed()) { - stmt = conn.createStatement(); - if (catalog == null || catalog.length() < 1) { - catalog = conn.getCatalog(); - } - stmt.executeUpdate("use " + catalog); - - DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); - // set up ColumnMetaDataList - List columnMetaDataList = new ArrayList<>(24); - columnMetaDataList.add(null); - columnMetaDataList.add(null); - // add TABLE_NAME - ColumnMetaData colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(3); - colMetaData.setColName("TABLE_NAME"); - colMetaData.setColSize(193); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add COLUMN_NAME - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(4); - colMetaData.setColName("COLUMN_NAME"); - colMetaData.setColSize(65); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add DATA_TYPE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(5); - colMetaData.setColName("DATA_TYPE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add TYPE_NAME - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(6); - colMetaData.setColName("TYPE_NAME"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add COLUMN_SIZE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(7); - colMetaData.setColName("COLUMN_SIZE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add BUFFER_LENGTH ,not used - columnMetaDataList.add(null); - // add DECIMAL_DIGITS - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(9); - colMetaData.setColName("DECIMAL_DIGITS"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add NUM_PREC_RADIX - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(10); - colMetaData.setColName("NUM_PREC_RADIX"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add NULLABLE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(11); - colMetaData.setColName("NULLABLE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - - resultSet.setColumnMetaDataList(columnMetaDataList); - - // set up rowDataList - ResultSet resultSet0 = stmt.executeQuery("describe " + tableNamePattern); - List rowDataList = new ArrayList<>(); - int index = 0; - while (resultSet0.next()) { - TSDBResultSetRowData rowData = new TSDBResultSetRowData(24); - // set TABLE_NAME - rowData.setString(2, tableNamePattern); - // set COLUMN_NAME - rowData.setString(3, resultSet0.getString(1)); - // set DATA_TYPE - String typeName = resultSet0.getString(2); - rowData.setInt(4, getDataType(typeName)); - // set TYPE_NAME - rowData.setString(5, typeName); - // set COLUMN_SIZE - int length = resultSet0.getInt(3); - rowData.setInt(6, getColumnSize(typeName, length)); - // set DECIMAL_DIGITS - rowData.setInt(8, getDecimalDigits(typeName)); - // set NUM_PREC_RADIX - rowData.setInt(9, 10); - // set NULLABLE - rowData.setInt(10, getNullable(index, typeName)); - rowDataList.add(rowData); - index++; - } - resultSet.setRowDataList(rowDataList); - -// GetColumnsResultSet getColumnsResultSet = new GetColumnsResultSet(resultSet0, catalog, schemaPattern, tableNamePattern, columnNamePattern); -// return getColumnsResultSet; -// DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); - return resultSet; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - - /*************************/ - -// return getEmptyResultSet(); - } - - private int getNullable(int index, String typeName) { - if (index == 0 && "TIMESTAMP".equals(typeName)) - return DatabaseMetaData.columnNoNulls; - return DatabaseMetaData.columnNullable; - } - - private int getColumnSize(String typeName, int length) { - switch (typeName) { - case "TIMESTAMP": - return 23; - - default: - return 0; - } - } - - private int getDecimalDigits(String typeName) { - switch (typeName) { - case "FLOAT": - return 5; - case "DOUBLE": - return 9; - default: - return 0; - } - } - - private int getDataType(String typeName) { - switch (typeName) { - case "TIMESTAMP": - return Types.TIMESTAMP; - case "INT": - return Types.INTEGER; - case "BIGINT": - return Types.BIGINT; - case "FLOAT": - return Types.FLOAT; - case "DOUBLE": - return Types.DOUBLE; - case "BINARY": - return Types.BINARY; - case "SMALLINT": - return Types.SMALLINT; - case "TINYINT": - return Types.TINYINT; - case "BOOL": - return Types.BOOLEAN; - case "NCHAR": - return Types.NCHAR; - default: - return Types.NULL; - } - } - - public ResultSet getColumnPrivileges(String catalog, String schema, String table, String columnNamePattern) - throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getTablePrivileges(String catalog, String schemaPattern, String tableNamePattern) - throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getBestRowIdentifier(String catalog, String schema, String table, int scope, boolean nullable) - throws SQLException { - return getEmptyResultSet(); + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getTableTypes(); } - public ResultSet getVersionColumns(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); + public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern, conn); } public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getImportedKeys(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getExportedKeys(String catalog, String schema, String table) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getCrossReference(String parentCatalog, String parentSchema, String parentTable, - String foreignCatalog, String foreignSchema, String foreignTable) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getTypeInfo() throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) - throws SQLException { - return getEmptyResultSet(); - } - - public boolean supportsResultSetType(int type) throws SQLException { - return false; - } - - public boolean supportsResultSetConcurrency(int type, int concurrency) throws SQLException { - return false; - } - - public boolean ownUpdatesAreVisible(int type) throws SQLException { - return false; - } - - public boolean ownDeletesAreVisible(int type) throws SQLException { - return false; - } - - public boolean ownInsertsAreVisible(int type) throws SQLException { - return false; - } - - public boolean othersUpdatesAreVisible(int type) throws SQLException { - return false; - } - - public boolean othersDeletesAreVisible(int type) throws SQLException { - return false; - } - - public boolean othersInsertsAreVisible(int type) throws SQLException { - return false; - } - - public boolean updatesAreDetected(int type) throws SQLException { - return false; - } - - public boolean deletesAreDetected(int type) throws SQLException { - return false; - } - - public boolean insertsAreDetected(int type) throws SQLException { - return false; - } - - public boolean supportsBatchUpdates() throws SQLException { - return false; - } - - public ResultSet getUDTs(String catalog, String schemaPattern, String typeNamePattern, int[] types) - throws SQLException { - return getEmptyResultSet(); - } - - public Connection getConnection() throws SQLException { - return null; - } - - public boolean supportsSavepoints() throws SQLException { - return false; - } - - public boolean supportsNamedParameters() throws SQLException { - return false; - } - - public boolean supportsMultipleOpenResults() throws SQLException { - return false; - } - - public boolean supportsGetGeneratedKeys() throws SQLException { - return false; - } - - public ResultSet getSuperTypes(String catalog, String schemaPattern, String typeNamePattern) throws SQLException { - return getEmptyResultSet(); + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getPrimaryKeys(catalog, schema, table, conn); } public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getAttributes(String catalog, String schemaPattern, String typeNamePattern, - String attributeNamePattern) throws SQLException { - return getEmptyResultSet(); - } - - public boolean supportsResultSetHoldability(int holdability) throws SQLException { - return false; - } - - public int getResultSetHoldability() throws SQLException { - return 0; + if (conn == null || conn.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getSuperTables(catalog, schemaPattern, tableNamePattern, conn); } - public int getDatabaseMajorVersion() throws SQLException { - return 0; - } - - public int getDatabaseMinorVersion() throws SQLException { - return 0; - } - - public int getJDBCMajorVersion() throws SQLException { - return 0; - } - - public int getJDBCMinorVersion() throws SQLException { - return 0; - } - - public int getSQLStateType() throws SQLException { - return 0; - } - - public boolean locatorsUpdateCopy() throws SQLException { - return false; - } - - public boolean supportsStatementPooling() throws SQLException { - return false; - } - - public RowIdLifetime getRowIdLifetime() throws SQLException { - return null; - } - - public ResultSet getSchemas(String catalog, String schemaPattern) throws SQLException { - return null; - } - - public boolean supportsStoredFunctionsUsingCallSyntax() throws SQLException { - return false; - } - - public boolean autoCommitFailureClosesAllResultSets() throws SQLException { - return false; - } - - public ResultSet getClientInfoProperties() throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getFunctions(String catalog, String schemaPattern, String functionNamePattern) - throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getFunctionColumns(String catalog, String schemaPattern, String functionNamePattern, - String columnNamePattern) throws SQLException { - return getEmptyResultSet(); - } - - public ResultSet getPseudoColumns(String catalog, String schemaPattern, String tableNamePattern, - String columnNamePattern) throws SQLException { - return getEmptyResultSet(); - } - - public boolean generatedKeyAlwaysReturned() throws SQLException { - return false; - } - - private ResultSet getEmptyResultSet() { - return new EmptyResultSet(); - } } \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 06f88cebfaa8aa90cc81506a98374ec8076ad82e..5f599df1300b3d6959e9a7b58c00a720d722e07b 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -37,13 +37,17 @@ import java.util.logging.Logger; * register it with the DriverManager. This means that a user can load and * register a driver by doing Class.forName("foo.bah.Driver") */ -public class TSDBDriver extends AbstractTaosDriver { +public class TSDBDriver extends AbstractDriver { @Deprecated private static final String URL_PREFIX1 = "jdbc:TSDB://"; private static final String URL_PREFIX = "jdbc:TAOS://"; + /** + * PRODUCT_NAME + */ + public static final String PROPERTY_KEY_PRODUCT_NAME = "productName"; /** * Key used to retrieve the host value from the properties instance passed to * the driver. @@ -90,46 +94,40 @@ public class TSDBDriver extends AbstractTaosDriver { * fetch data from native function in a batch model */ public static final String PROPERTY_KEY_BATCH_LOAD = "batchfetch"; - + private TSDBDatabaseMetaData dbMetaData = null; static { try { java.sql.DriverManager.registerDriver(new TSDBDriver()); - } catch (SQLException E) { - throw new RuntimeException(TSDBConstants.WrapErrMsg("can't register tdengine jdbc driver!")); + } catch (SQLException e) { + throw TSDBError.createRuntimeException(TSDBErrorNumbers.ERROR_CANNOT_REGISTER_JNI_DRIVER, e); } } public Connection connect(String url, Properties info) throws SQLException { if (url == null) - throw new SQLException(TSDBConstants.WrapErrMsg("url is not set!")); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_URL_NOT_SET); if (!acceptsURL(url)) return null; - Properties props = null; - if ((props = parseURL(url, info)) == null) { + Properties props = parseURL(url, info); + if (props == null) { return null; } - //load taos.cfg start - loadTaosConfig(info); try { TSDBJNIConnector.init((String) props.get(PROPERTY_KEY_CONFIG_DIR), (String) props.get(PROPERTY_KEY_LOCALE), (String) props.get(PROPERTY_KEY_CHARSET), (String) props.get(PROPERTY_KEY_TIME_ZONE)); - Connection newConn = new TSDBConnection(props, this.dbMetaData); - return newConn; + return new TSDBConnection(props, this.dbMetaData); } catch (SQLWarning sqlWarning) { sqlWarning.printStackTrace(); - Connection newConn = new TSDBConnection(props, this.dbMetaData); - return newConn; + return new TSDBConnection(props, this.dbMetaData); } catch (SQLException sqlEx) { throw sqlEx; } catch (Exception ex) { - SQLException sqlEx = new SQLException("SQLException:" + ex.toString()); - sqlEx.initCause(ex); - throw sqlEx; + throw new SQLException("SQLException:" + ex.toString(), ex); } } @@ -141,8 +139,8 @@ public class TSDBDriver extends AbstractTaosDriver { */ public boolean acceptsURL(String url) throws SQLException { if (url == null) - throw new SQLException(TSDBConstants.WrapErrMsg("url is null")); - return (url != null && url.length() > 0 && url.trim().length() > 0) && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_URL_NOT_SET); + return url.length() > 0 && url.trim().length() > 0 && (url.startsWith(URL_PREFIX) || url.startsWith(URL_PREFIX1)); } public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { @@ -179,18 +177,18 @@ public class TSDBDriver extends AbstractTaosDriver { while (queryParams.hasMoreElements()) { String oneToken = queryParams.nextToken(); String[] pair = oneToken.split("="); - + if ((pair[0] != null && pair[0].trim().length() > 0) && (pair[1] != null && pair[1].trim().length() > 0)) { urlProps.setProperty(pair[0].trim(), pair[1].trim()); } } } - + // parse Product Name String dbProductName = url.substring(0, beginningOfSlashes); dbProductName = dbProductName.substring(dbProductName.indexOf(":") + 1); dbProductName = dbProductName.substring(0, dbProductName.indexOf(":")); - + // parse database name url = url.substring(beginningOfSlashes + 2); int indexOfSlash = url.indexOf("/"); @@ -200,7 +198,7 @@ public class TSDBDriver extends AbstractTaosDriver { } url = url.substring(0, indexOfSlash); } - + // parse port int indexOfColon = url.indexOf(":"); if (indexOfColon != -1) { @@ -209,12 +207,12 @@ public class TSDBDriver extends AbstractTaosDriver { } url = url.substring(0, indexOfColon); } - + if (url != null && url.length() > 0 && url.trim().length() > 0) { urlProps.setProperty(TSDBDriver.PROPERTY_KEY_HOST, url); } - - this.dbMetaData = new TSDBDatabaseMetaData(dbProductName, urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER)); + + this.dbMetaData = new TSDBDatabaseMetaData(urlForMeta, urlProps.getProperty(TSDBDriver.PROPERTY_KEY_USER)); return urlProps; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java new file mode 100644 index 0000000000000000000000000000000000000000..90967b3620fdaccf69f253c8f8927c067b6221fd --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBError.java @@ -0,0 +1,84 @@ +package com.taosdata.jdbc; + +import java.sql.SQLClientInfoException; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.sql.SQLWarning; +import java.util.HashMap; +import java.util.Map; + +public class TSDBError { + private static Map TSDBErrorMap = new HashMap<>(); + + static { + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED, "connection already closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD, "this operation is NOT supported currently!"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_VARIABLE, "invalid variables"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED, "statement is closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED, "resultSet is closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_BATCH_IS_EMPTY, "Batch is empty!"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY, "Can not issue data manipulation statements with executeQuery()"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEUPDATE, "Can not issue SELECT via executeUpdate()"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "invalid sql for executeQuery: (?)"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_DATABASE_NOT_SPECIFIED_OR_AVAILABLE, "Database not specified or available"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "invalid sql for executeUpdate: (?)"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE, "invalid sql for execute: (?)"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE, "parameter index out of range"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED, "connection already closed"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE, "unknown sql type in tdengine"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_CANNOT_REGISTER_JNI_DRIVER, "can't register JDBC-JNI driver"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_CANNOT_REGISTER_RESTFUL_DRIVER, "can't register JDBC-RESTful driver"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_URL_NOT_SET, "url is not set"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_INVALID_SQL, "invalid sql"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE, "numeric value out of range"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE, "unknown taos type in tdengine"); + + /**************************************************/ + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNKNOWN, "unknown error"); + /**************************************************/ + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_SUBSCRIBE_FAILED, "failed to create subscription"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_UNSUPPORTED_ENCODING, "Unsupported encoding"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_TDENGINE_ERROR, "internal error of database"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL, "JNI connection is NULL"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL, "JNI result set is NULL"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0, "invalid num of fields"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_SQL_NULL, "empty sql string"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_FETCH_END, "fetch to the end of resultSet"); + TSDBErrorMap.put(TSDBErrorNumbers.ERROR_JNI_OUT_OF_MEMORY, "JNI alloc memory failed"); + } + + public static SQLException createSQLException(int errorCode) { + String message; + if (TSDBErrorNumbers.contains(errorCode)) + message = TSDBErrorMap.get(errorCode); + else + message = TSDBErrorMap.get(TSDBErrorNumbers.ERROR_UNKNOWN); + return createSQLException(errorCode, message); + } + + public static SQLException createSQLException(int errorCode, String message) { + // throw SQLFeatureNotSupportedException + if (errorCode == TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD) + return new SQLFeatureNotSupportedException(message, "", errorCode); + // throw SQLClientInfoException + if (errorCode == TSDBErrorNumbers.ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED) + return new SQLClientInfoException(message, null); + + if (errorCode > 0x2300 && errorCode < 0x2350) + // JDBC exception's error number is less than 0x2350 + return new SQLException("ERROR (" + Integer.toHexString(errorCode) + "): " + message, "", errorCode); + if (errorCode > 0x2350 && errorCode < 0x2400) + // JNI exception's error number is large than 0x2350 + return new SQLException("JNI ERROR (" + Integer.toHexString(errorCode) + "): " + message, "", errorCode); + return new SQLException("TDengine ERROR (" + Integer.toHexString(errorCode) + "): " + message, "", errorCode); + } + + public static RuntimeException createRuntimeException(int errorCode, Throwable t) { + String message = TSDBErrorMap.get(errorCode); + return new RuntimeException("ERROR (" + Integer.toHexString(errorCode) + "): " + message, t); + } + + public static SQLWarning createSQLWarning(String message) { + return new SQLWarning(message); + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java new file mode 100644 index 0000000000000000000000000000000000000000..c978bb3a2e4a1b9dbd264268f057f2b6c735250a --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java @@ -0,0 +1,84 @@ +package com.taosdata.jdbc; + +import java.util.HashSet; + +public class TSDBErrorNumbers { + + public static final int ERROR_CONNECTION_CLOSED = 0x2301; // connection already closed + public static final int ERROR_UNSUPPORTED_METHOD = 0x2302; //this operation is NOT supported currently! + public static final int ERROR_INVALID_VARIABLE = 0x2303; //invalid variables + public static final int ERROR_STATEMENT_CLOSED = 0x2304; //statement already closed + public static final int ERROR_RESULTSET_CLOSED = 0x2305; //resultSet is closed + public static final int ERROR_BATCH_IS_EMPTY = 0x2306; //Batch is empty! + public static final int ERROR_INVALID_WITH_EXECUTEQUERY = 0x2307; //Can not issue data manipulation statements with executeQuery() + public static final int ERROR_INVALID_WITH_EXECUTEUPDATE = 0x2308; //Can not issue SELECT via executeUpdate() + public static final int ERROR_INVALID_FOR_EXECUTE_QUERY = 0x2309; //not a valid sql for executeQuery: (SQL) + public static final int ERROR_DATABASE_NOT_SPECIFIED_OR_AVAILABLE = 0x230a; //Database not specified or available + public static final int ERROR_INVALID_FOR_EXECUTE_UPDATE = 0x230b; //not a valid sql for executeUpdate: (SQL) + public static final int ERROR_INVALID_FOR_EXECUTE = 0x230c; //not a valid sql for execute: (SQL) + public static final int ERROR_PARAMETER_INDEX_OUT_RANGE = 0x230d; // parameter index out of range + public static final int ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED = 0x230e; // connection already closed + public static final int ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE = 0x230f; //unknown sql type in tdengine + public static final int ERROR_CANNOT_REGISTER_JNI_DRIVER = 0x2310; // can't register JDBC-JNI driver + public static final int ERROR_CANNOT_REGISTER_RESTFUL_DRIVER = 0x2311; // can't register JDBC-RESTful driver + public static final int ERROR_URL_NOT_SET = 0x2312; // url is not set + public static final int ERROR_INVALID_SQL = 0x2313; // invalid sql + public static final int ERROR_NUMERIC_VALUE_OUT_OF_RANGE = 0x2314; // numeric value out of range + public static final int ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE = 0x2315; //unknown taos type in tdengine + + public static final int ERROR_UNKNOWN = 0x2350; //unknown error + + public static final int ERROR_SUBSCRIBE_FAILED = 0x2351; // failed to create subscription + public static final int ERROR_UNSUPPORTED_ENCODING = 0x2352; // Unsupported encoding + public static final int ERROR_JNI_TDENGINE_ERROR = 0x2353; // internal error of database + public static final int ERROR_JNI_CONNECTION_NULL = 0x2354; // JNI connection is NULL + public static final int ERROR_JNI_RESULT_SET_NULL = 0x2355; // invalid JNI result set + public static final int ERROR_JNI_NUM_OF_FIELDS_0 = 0x2356; // invalid num of fields + public static final int ERROR_JNI_SQL_NULL = 0x2357; // empty sql string + public static final int ERROR_JNI_FETCH_END = 0x2358; // fetch to the end of resultSet + public static final int ERROR_JNI_OUT_OF_MEMORY = 0x2359; // JNI alloc memory failed + + private static final HashSet errorNumbers; + + static { + errorNumbers = new HashSet(); + errorNumbers.add(ERROR_CONNECTION_CLOSED); + errorNumbers.add(ERROR_UNSUPPORTED_METHOD); + errorNumbers.add(ERROR_INVALID_VARIABLE); + errorNumbers.add(ERROR_STATEMENT_CLOSED); + errorNumbers.add(ERROR_RESULTSET_CLOSED); + errorNumbers.add(ERROR_INVALID_WITH_EXECUTEQUERY); + errorNumbers.add(ERROR_INVALID_WITH_EXECUTEUPDATE); + errorNumbers.add(ERROR_INVALID_FOR_EXECUTE_QUERY); + errorNumbers.add(ERROR_DATABASE_NOT_SPECIFIED_OR_AVAILABLE); + errorNumbers.add(ERROR_INVALID_FOR_EXECUTE_UPDATE); + errorNumbers.add(ERROR_INVALID_FOR_EXECUTE); + errorNumbers.add(ERROR_PARAMETER_INDEX_OUT_RANGE); + errorNumbers.add(ERROR_SQLCLIENT_EXCEPTION_ON_CONNECTION_CLOSED); + errorNumbers.add(ERROR_UNKNOWN_SQL_TYPE_IN_TDENGINE); + errorNumbers.add(ERROR_CANNOT_REGISTER_JNI_DRIVER); + errorNumbers.add(ERROR_CANNOT_REGISTER_RESTFUL_DRIVER); + errorNumbers.add(ERROR_URL_NOT_SET); + errorNumbers.add(ERROR_INVALID_SQL); + errorNumbers.add(ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + errorNumbers.add(ERROR_UNKNOWN_TAOS_TYPE_IN_TDENGINE); + + /*****************************************************/ + errorNumbers.add(ERROR_SUBSCRIBE_FAILED); + errorNumbers.add(ERROR_UNSUPPORTED_ENCODING); + errorNumbers.add(ERROR_JNI_TDENGINE_ERROR); + errorNumbers.add(ERROR_JNI_CONNECTION_NULL); + errorNumbers.add(ERROR_JNI_RESULT_SET_NULL); + errorNumbers.add(ERROR_JNI_NUM_OF_FIELDS_0); + errorNumbers.add(ERROR_JNI_SQL_NULL); + errorNumbers.add(ERROR_JNI_FETCH_END); + errorNumbers.add(ERROR_JNI_OUT_OF_MEMORY); + } + + private TSDBErrorNumbers() { + } + + public static boolean contains(int errorNumber) { + return errorNumbers.contains(errorNumber); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index f918463439fba293171827001acf6930ab271b81..5e3ffffa4fe7326b8c1e89ac91a37e95c346784f 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -1,76 +1,65 @@ -/*************************************************************************** +/** + * ************************************************************************* * Copyright (c) 2019 TAOS Data, Inc. - * + *

    * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation. - * + *

    * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. - * + *

    * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - *****************************************************************************/ + * *************************************************************************** + */ package com.taosdata.jdbc; +import com.taosdata.jdbc.utils.TaosInfo; + import java.sql.SQLException; import java.sql.SQLWarning; import java.util.List; +/** + * JNI connector + */ public class TSDBJNIConnector { private static volatile Boolean isInitialized = false; + private TaosInfo taosInfo = TaosInfo.getInstance(); + // Connection pointer used in C + private long taos = TSDBConstants.JNI_NULL_POINTER; + // result set status in current connection + private boolean isResultsetClosed = true; + private int affectedRows = -1; + static { System.loadLibrary("taos"); System.out.println("java.library.path:" + System.getProperty("java.library.path")); } - /** - * Connection pointer used in C - */ - private long taos = TSDBConstants.JNI_NULL_POINTER; - - /** - * Result set pointer for the current connection - */ - private long taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - - /** - * result set status in current connection - */ - private boolean isResultsetClosed = true; - private int affectedRows = -1; - - /** - * Whether the connection is closed - */ public boolean isClosed() { return this.taos == TSDBConstants.JNI_NULL_POINTER; } - /** - * Returns the status of last result set in current connection - */ public boolean isResultsetClosed() { return this.isResultsetClosed; } - /** - * Initialize static variables in JNI to optimize performance - */ public static void init(String configDir, String locale, String charset, String timezone) throws SQLWarning { synchronized (isInitialized) { if (!isInitialized) { initImp(configDir); if (setOptions(0, locale) < 0) { - throw new SQLWarning(TSDBConstants.WrapErrMsg("Failed to set locale: " + locale + ". System default will be used.")); + throw TSDBError.createSQLWarning("Failed to set locale: " + locale + ". System default will be used."); } if (setOptions(1, charset) < 0) { - throw new SQLWarning(TSDBConstants.WrapErrMsg("Failed to set charset: " + charset + ". System default will be used.")); + throw TSDBError.createSQLWarning("Failed to set charset: " + charset + ". System default will be used."); } if (setOptions(2, timezone) < 0) { - throw new SQLWarning(TSDBConstants.WrapErrMsg("Failed to set timezone: " + timezone + ". System default will be used.")); + throw TSDBError.createSQLWarning("Failed to set timezone: " + timezone + ". System default will be used."); } isInitialized = true; TaosGlobalConfig.setCharset(getTsCharset()); @@ -84,22 +73,19 @@ public class TSDBJNIConnector { public static native String getTsCharset(); - /** - * Get connection pointer - * - * @throws SQLException - */ public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException { if (this.taos != TSDBConstants.JNI_NULL_POINTER) { - this.closeConnectionImp(this.taos); +// this.closeConnectionImp(this.taos); + closeConnection(); this.taos = TSDBConstants.JNI_NULL_POINTER; } this.taos = this.connectImp(host, port, dbName, user, password); if (this.taos == TSDBConstants.JNI_NULL_POINTER) { - throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(0L)), "", this.getErrCode(0l)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); } - + // invoke connectImp only here + taosInfo.conn_open_increment(); return true; } @@ -113,31 +99,43 @@ public class TSDBJNIConnector { public long executeQuery(String sql) throws SQLException { // close previous result set if the user forgets to invoke the // free method to close previous result set. - if (!this.isResultsetClosed) { - freeResultSet(taosResultSetPointer); - } +// if (!this.isResultsetClosed) { +// freeResultSet(taosResultSetPointer); +// } Long pSql = 0l; try { pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); + taosInfo.stmt_count_increment(); } catch (Exception e) { e.printStackTrace(); this.freeResultSetImp(this.taos, pSql); - throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding")); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_ENCODING); + } + if (pSql == TSDBConstants.JNI_CONNECTION_NULL) { + this.freeResultSetImp(this.taos, pSql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } + if (pSql == TSDBConstants.JNI_SQL_NULL) { + this.freeResultSetImp(this.taos, pSql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_SQL_NULL); + } + if (pSql == TSDBConstants.JNI_OUT_OF_MEMORY) { + this.freeResultSetImp(this.taos, pSql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_OUT_OF_MEMORY); } int code = this.getErrCode(pSql); - if (code != 0) { + if (code != TSDBConstants.JNI_SUCCESS) { affectedRows = -1; String msg = this.getErrMsg(pSql); - this.freeResultSetImp(this.taos, pSql); - throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code); + throw TSDBError.createSQLException(code, msg); } // Try retrieving result set for the executed SQL using the current connection pointer. - taosResultSetPointer = this.getResultSetImp(this.taos, pSql); - isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER); + pSql = this.getResultSetImp(this.taos, pSql); + isResultsetClosed = (pSql == TSDBConstants.JNI_NULL_POINTER); return pSql; } @@ -162,14 +160,6 @@ public class TSDBJNIConnector { private native String getErrMsgImp(long pSql); - /** - * Get resultset pointer - * Each connection should have a single open result set at a time - */ - public long getResultSet() { - return taosResultSetPointer; - } - private native long getResultSetImp(long connection, long pSql); public boolean isUpdateQuery(long pSql) { @@ -181,16 +171,16 @@ public class TSDBJNIConnector { /** * Free resultset operation from C to release resultset pointer by JNI */ - public int freeResultSet(long result) { + public int freeResultSet(long pSql) { int res = TSDBConstants.JNI_SUCCESS; - if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - throw new RuntimeException("Invalid result set pointer"); - } +// if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { +// throw new RuntimeException("Invalid result set pointer"); +// } - if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - res = this.freeResultSetImp(this.taos, result); - taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - } +// if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { + res = this.freeResultSetImp(this.taos, pSql); +// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; +// } isResultsetClosed = true; return res; @@ -200,15 +190,15 @@ public class TSDBJNIConnector { * Close the open result set which is associated to the current connection. If the result set is already * closed, return 0 for success. */ - public int freeResultSet() { - int resCode = TSDBConstants.JNI_SUCCESS; - if (!isResultsetClosed) { - resCode = this.freeResultSetImp(this.taos, this.taosResultSetPointer); - taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - isResultsetClosed = true; - } - return resCode; - } +// public int freeResultSet() { +// int resCode = TSDBConstants.JNI_SUCCESS; +// if (!isResultsetClosed) { +// resCode = this.freeResultSetImp(this.taos, this.taosResultSetPointer); +// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; +// isResultsetClosed = true; +// } +// return resCode; +// } private native int freeResultSetImp(long connection, long result); @@ -244,10 +234,11 @@ public class TSDBJNIConnector { private native int fetchRowImp(long connection, long resultSet, TSDBResultSetRowData rowData); public int fetchBlock(long resultSet, TSDBResultSetBlockData blockData) { - return this.fetchBlockImp(this.taos, resultSet, blockData); + return this.fetchBlockImp(this.taos, resultSet, blockData); } - + private native int fetchBlockImp(long connection, long resultSet, TSDBResultSetBlockData blockData); + /** * Execute close operation from C to release connection pointer by JNI * @@ -256,12 +247,14 @@ public class TSDBJNIConnector { public void closeConnection() throws SQLException { int code = this.closeConnectionImp(this.taos); if (code < 0) { - throw new SQLException(TSDBConstants.FixErrMsg(code), "", this.getErrCode(0l)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); } else if (code == 0) { this.taos = TSDBConstants.JNI_NULL_POINTER; } else { throw new SQLException("Undefined error code returned by TDengine when closing a connection"); } + // invoke closeConnectionImpl only here + taosInfo.connect_close_increment(); } private native int closeConnectionImp(long connection); @@ -299,8 +292,7 @@ public class TSDBJNIConnector { * Validate if a create table sql statement is correct without actually creating that table */ public boolean validateCreateTableSql(String sql) { - long connection = taos; - int res = validateCreateTableSqlImp(connection, sql.getBytes()); + int res = validateCreateTableSqlImp(taos, sql.getBytes()); return res != 0 ? false : true; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java index d9227523d4ac623d23d85be6376a1530ca606751..9ee23e3a265298632a953b4a3feedeaf933905ca 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBParameterMetaData.java @@ -1,75 +1,8 @@ -/*************************************************************************** - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - *****************************************************************************/ package com.taosdata.jdbc; -import java.sql.ParameterMetaData; -import java.sql.SQLException; +public class TSDBParameterMetaData extends AbstractParameterMetaData { -public class TSDBParameterMetaData implements ParameterMetaData { - @Override - public int getParameterCount() throws SQLException { - return 0; - } - - @Override - public int isNullable(int param) throws SQLException { - return 0; - } - - @Override - public boolean isSigned(int param) throws SQLException { - return false; - } - - @Override - public int getPrecision(int param) throws SQLException { - return 0; - } - - @Override - public int getScale(int param) throws SQLException { - return 0; - } - - @Override - public int getParameterType(int param) throws SQLException { - return 0; - } - - @Override - public String getParameterTypeName(int param) throws SQLException { - return null; - } - - @Override - public String getParameterClassName(int param) throws SQLException { - return null; - } - - @Override - public int getParameterMode(int param) throws SQLException { - return 0; - } - - @Override - public T unwrap(Class iface) throws SQLException { - return null; - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return false; + public TSDBParameterMetaData(Object[] parameters) { + super(parameters); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java index 230943fd53ae8fa36b9de40f851a5c263c052f0d..728b537f715e69b4c79e3e67d2b9672c2e3d0714 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBPreparedStatement.java @@ -18,6 +18,7 @@ import java.io.InputStream; import java.io.Reader; import java.math.BigDecimal; import java.net.URL; +import java.nio.charset.Charset; import java.sql.*; import java.util.ArrayList; import java.util.Calendar; @@ -30,36 +31,49 @@ import java.util.regex.Pattern; * compatibility needs. */ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStatement { - protected String rawSql; - protected String sql; - protected ArrayList parameters = new ArrayList(); + + private String rawSql; + private String sql; + // private ArrayList parameters = new ArrayList<>(); + private Object[] parameters; + private boolean isPrepared; //start with insert or import and is case-insensitive private static Pattern savePattern = Pattern.compile("(?i)^\\s*(insert|import)"); - // is insert or import private boolean isSaved; - private SavedPreparedStatement savedPreparedStatement; + // private SavedPreparedStatement savedPreparedStatement; + private volatile TSDBParameterMetaData parameterMetaData; TSDBPreparedStatement(TSDBConnection connection, TSDBJNIConnector connecter, String sql) { super(connection, connecter); init(sql); + + if (sql.contains("?")) { + int parameterCnt = 0; + for (int i = 0; i < sql.length(); i++) { + if ('?' == sql.charAt(i)) { + parameterCnt++; + } + } + parameters = new Object[parameterCnt]; + this.isPrepared = true; + } } private void init(String sql) { this.rawSql = sql; preprocessSql(); +// this.isSaved = isSavedSql(this.rawSql); +// if (this.isSaved) { +// try { +// this.savedPreparedStatement = new SavedPreparedStatement(this.rawSql, this); +// } catch (SQLException e) { +// e.printStackTrace(); +// } +// } - this.isSaved = isSavedSql(this.rawSql); - if (this.isSaved) { - - try { - this.savedPreparedStatement = new SavedPreparedStatement(this.rawSql, this); - } catch (SQLException e) { - e.printStackTrace(); - } - } } /** @@ -75,23 +89,11 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat @Override public int[] executeBatch() throws SQLException { - if (isSaved) { - return this.savedPreparedStatement.executeBatch(); - } else { - return super.executeBatch(); - } - } - - public ArrayList getParameters() { - return parameters; - } - - public void setParameters(ArrayList parameters) { - this.parameters = parameters; - } - - public String getRawSql() { - return rawSql; +// if (isSaved) { +// return this.savedPreparedStatement.executeBatch(); +// } else { + return super.executeBatch(); +// } } /* @@ -100,7 +102,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat * order to process those supported SQLs. */ private void preprocessSql() { - /***** For processing some of Spark SQLs*****/ // should replace it first this.rawSql = this.rawSql.replaceAll("or (.*) is null", ""); @@ -149,7 +150,6 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat rawSql = rawSql.replace(matcher.group(1), tableFullName); } /***** for inner queries *****/ - } /** @@ -157,322 +157,507 @@ public class TSDBPreparedStatement extends TSDBStatement implements PreparedStat * * @return a string of the native sql statement for TSDB */ - private String getNativeSql() { - this.sql = this.rawSql; - for (int i = 0; i < parameters.size(); ++i) { - Object para = parameters.get(i); +// private String getNativeSql(String rawSql) { +// for (int i = 0; i < parameters.length; i++) { +// Object para = parameters[i]; +// if (para != null) { +// String paraStr = para.toString(); +// if (para instanceof Timestamp || para instanceof String) { +// paraStr = "'" + paraStr + "'"; +// } +// this.sql = this.sql.replaceFirst("[?]", paraStr); +// } else { +// this.sql = this.sql.replaceFirst("[?]", "NULL"); +// } +// } +// parameters = new Object[parameters.length]; +// return sql; +// } + + private String getNativeSql(String rawSql) throws SQLException { + String sql = rawSql; + for (int i = 0; i < parameters.length; ++i) { + Object para = parameters[i]; if (para != null) { - String paraStr = para.toString(); - if (para instanceof Timestamp || para instanceof String) { + String paraStr; + if (para instanceof byte[]) { + paraStr = new String((byte[]) para, Charset.forName("UTF-8")); + } else { + paraStr = para.toString(); + } + // if para is timestamp or String or byte[] need to translate ' character + if (para instanceof Timestamp || para instanceof String || para instanceof byte[]) { + paraStr = paraStr.replaceAll("'", "\\\\\\\\'"); paraStr = "'" + paraStr + "'"; } - this.sql = this.sql.replaceFirst("[?]", paraStr); + sql = sql.replaceFirst("[?]", paraStr); } else { - this.sql = this.sql.replaceFirst("[?]", "NULL"); + sql = sql.replaceFirst("[?]", "NULL"); } } - parameters.clear(); + clearParameters(); return sql; } @Override public ResultSet executeQuery() throws SQLException { - if (isSaved) { - this.savedPreparedStatement.executeBatchInternal(); - return null; - } else { - return super.executeQuery(getNativeSql()); - } +// if (isSaved) { +// this.savedPreparedStatement.executeBatchInternal(); +// return null; +// } else { + + if (!isPrepared) + return executeQuery(this.rawSql); + + final String sql = getNativeSql(this.rawSql); + return executeQuery(sql); +// } } @Override public int executeUpdate() throws SQLException { - if (isSaved) { - return this.savedPreparedStatement.executeBatchInternal(); - } else { - return super.executeUpdate(getNativeSql()); +// if (isSaved) { +// return this.savedPreparedStatement.executeBatchInternal(); +// } else { + if (!isPrepared) + return executeUpdate(this.rawSql); + String sql = getNativeSql(this.rawSql); + return executeUpdate(sql); +// } + } + + private boolean isSupportedSQLType(int sqlType) { + switch (sqlType) { + case Types.TIMESTAMP: + case Types.INTEGER: + case Types.BIGINT: + case Types.FLOAT: + case Types.DOUBLE: + case Types.SMALLINT: + case Types.TINYINT: + case Types.BOOLEAN: + case Types.BINARY: + case Types.NCHAR: + return true; + default: + return false; } } @Override public void setNull(int parameterIndex, int sqlType) throws SQLException { - setObject(parameterIndex, new String("NULL")); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!isSupportedSQLType(sqlType) || parameterIndex < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); +// if (parameterIndex >= parameters.size()) +// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_BOUNDARY); + + setObject(parameterIndex, "NULL"); } @Override public void setBoolean(int parameterIndex, boolean x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex, x); } @Override public void setByte(int parameterIndex, byte x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex,x); } @Override public void setShort(int parameterIndex, short x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setInt(int parameterIndex, int x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setLong(int parameterIndex, long x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setFloat(int parameterIndex, float x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setDouble(int parameterIndex, double x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setString(int parameterIndex, String x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setBytes(int parameterIndex, byte[] x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex,x); } @Override public void setDate(int parameterIndex, Date x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setTime(int parameterIndex, Time x) throws SQLException { - setObject(parameterIndex, x); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); setObject(parameterIndex, x); } @Override public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void clearParameters() throws SQLException { - parameters.clear(); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + +// parameters.clear(); + parameters = new Object[parameters.length]; } @Override public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setObject(int parameterIndex, Object x) throws SQLException { - if (isSaved) { - this.savedPreparedStatement.setParam(parameterIndex, x); - } else { - parameters.add(x); - } +// if (isSaved) { +// this.savedPreparedStatement.setParam(parameterIndex, x); +// } else { + if (parameterIndex < 1 && parameterIndex >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + parameters[parameterIndex - 1] = x; +// parameters.add(x); +// } } @Override public boolean execute() throws SQLException { - if (isSaved) { - int result = this.savedPreparedStatement.executeBatchInternal(); - return result > 0; - } else { - return super.execute(getNativeSql()); - } +// if (isSaved) { +// int result = this.savedPreparedStatement.executeBatchInternal(); +// return result > 0; +// } else { + if (!isPrepared) + return execute(this.rawSql); + + final String sql = getNativeSql(this.rawSql); + + return execute(sql); +// } } @Override public void addBatch() throws SQLException { - if (isSaved) { - this.savedPreparedStatement.addBatch(); - } else { +// if (isSaved) { +// this.savedPreparedStatement.addBatch(); +// } else { + if (this.batchedArgs == null) { + batchedArgs = new ArrayList<>(); + } - if (this.batchedArgs == null) { - batchedArgs = new ArrayList(); - } - super.addBatch(getNativeSql()); + if (!isPrepared) { + addBatch(this.rawSql); + } else { + String sql = this.getConnection().nativeSQL(this.rawSql); + addBatch(sql); } +// } } @Override public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setRef(int parameterIndex, Ref x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBlob(int parameterIndex, Blob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setClob(int parameterIndex, Clob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setArray(int parameterIndex, Array x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public ResultSetMetaData getMetaData() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); +// return this.getResultSet().getMetaData(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // TODO: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setURL(int parameterIndex, URL x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public ParameterMetaData getParameterMetaData() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + if (parameterMetaData == null) { + this.parameterMetaData = new TSDBParameterMetaData(parameters); + } + return this.parameterMetaData; } @Override public void setRowId(int parameterIndex, RowId x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNString(int parameterIndex, String value) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNClob(int parameterIndex, NClob value) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + //TODO: + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setClob(int parameterIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } @Override public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public void setNClob(int parameterIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index 84a3f58f4692a99737af7d93c5578fc7a5a09c27..a20ddaa836535b2249ecc1a05e74a7903b9caeeb 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -14,1086 +14,460 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.io.InputStream; -import java.io.Reader; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.Shorts; + import java.math.BigDecimal; -import java.net.URL; -import java.sql.Array; -import java.sql.Blob; -import java.sql.Clob; -import java.sql.Date; -import java.sql.NClob; -import java.sql.Ref; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.RowId; -import java.sql.SQLException; -import java.sql.SQLWarning; -import java.sql.SQLXML; -import java.sql.Statement; -import java.sql.Time; -import java.sql.Timestamp; +import java.sql.*; import java.util.ArrayList; import java.util.Calendar; -import java.util.Iterator; import java.util.List; -import java.util.Map; - -@SuppressWarnings("unused") -public class TSDBResultSet implements ResultSet { - private TSDBJNIConnector jniConnector = null; - - private long resultSetPointer = 0L; - private List columnMetaDataList = new ArrayList(); - - private TSDBResultSetRowData rowData; - private TSDBResultSetBlockData blockData; - - private boolean batchFetch = false; - private boolean lastWasNull = false; - private final int COLUMN_INDEX_START_VALUE = 1; - - private int rowIndex = 0; - - public TSDBJNIConnector getJniConnector() { - return jniConnector; - } - - public void setJniConnector(TSDBJNIConnector jniConnector) { - this.jniConnector = jniConnector; - } - - public long getResultSetPointer() { - return resultSetPointer; - } - - public void setResultSetPointer(long resultSetPointer) { - this.resultSetPointer = resultSetPointer; - } - - public void setBatchFetch(boolean batchFetch) { - this.batchFetch = batchFetch; - } - - public Boolean getBatchFetch() { - return this.batchFetch; - } - - public List getColumnMetaDataList() { - return columnMetaDataList; - } - - public void setColumnMetaDataList(List columnMetaDataList) { - this.columnMetaDataList = columnMetaDataList; - } - - public TSDBResultSetRowData getRowData() { - return rowData; - } - - public void setRowData(TSDBResultSetRowData rowData) { - this.rowData = rowData; - } - - public boolean isLastWasNull() { - return lastWasNull; - } - - public void setLastWasNull(boolean lastWasNull) { - this.lastWasNull = lastWasNull; - } - - public TSDBResultSet() { - } - - public TSDBResultSet(TSDBJNIConnector connector, long resultSetPointer) throws SQLException { - this.jniConnector = connector; - this.resultSetPointer = resultSetPointer; - int code = this.jniConnector.getSchemaMetaData(this.resultSetPointer, this.columnMetaDataList); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } - - this.rowData = new TSDBResultSetRowData(this.columnMetaDataList.size()); - this.blockData = new TSDBResultSetBlockData(this.columnMetaDataList, this.columnMetaDataList.size()); - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean next() throws SQLException { - if (this.getBatchFetch()) { - if (this.blockData.forward()) { - return true; - } - - int code = this.jniConnector.fetchBlock(this.resultSetPointer, this.blockData); - this.blockData.reset(); - - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } else if (code == TSDBConstants.JNI_FETCH_END) { - return false; - } - - return true; - } else { - if (rowData != null) { - this.rowData.clear(); - } - - int code = this.jniConnector.fetchRow(this.resultSetPointer, this.rowData); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_NUM_OF_FIELDS_0)); - } else if (code == TSDBConstants.JNI_FETCH_END) { - return false; - } else { - return true; - } - } - } - - public void close() throws SQLException { - if (this.jniConnector != null) { - int code = this.jniConnector.freeResultSet(this.resultSetPointer); - if (code == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_RESULT_SET_NULL)); - } - } - } - - public boolean wasNull() throws SQLException { - return this.lastWasNull; - } - - public String getString(int columnIndex) throws SQLException { - String res = null; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getString(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getString(colIndex); - } - } - - public boolean getBoolean(int columnIndex) throws SQLException { - boolean res = false; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getBoolean(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - } else { - return this.blockData.getBoolean(colIndex); - } - - return res; - } - - public byte getByte(int columnIndex) throws SQLException { - byte res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = (byte) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (byte) this.blockData.getInt(colIndex); - } - } - - public short getShort(int columnIndex) throws SQLException { - short res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = (short) this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (short) this.blockData.getInt(colIndex); - } - } - - public int getInt(int columnIndex) throws SQLException { - int res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getInt(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getInt(colIndex); - } - - } - - public long getLong(int columnIndex) throws SQLException { - long res = 0l; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getLong(colIndex); - } - } - - public float getFloat(int columnIndex) throws SQLException { - float res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getFloat(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return (float) this.blockData.getDouble(colIndex); - } - } - - public double getDouble(int columnIndex) throws SQLException { - double res = 0; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getDouble(colIndex, this.columnMetaDataList.get(colIndex).getColType()); - } - return res; - } else { - return this.blockData.getDouble(colIndex); - } - } - - /* - * (non-Javadoc) - * - * @see java.sql.ResultSet#getBigDecimal(int, int) - * - * @deprecated Use {@code getBigDecimal(int columnIndex)} or {@code - * getBigDecimal(String columnLabel)} - */ - @Deprecated - public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - return new BigDecimal(getLong(columnIndex)); - } - - public byte[] getBytes(int columnIndex) throws SQLException { - return getString(columnIndex).getBytes(); - } - - public Date getDate(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Time getTime(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Timestamp getTimestamp(int columnIndex) throws SQLException { - Timestamp res = null; - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - if (!lastWasNull) { - res = this.rowData.getTimestamp(colIndex); - } - return res; - } else { - return this.blockData.getTimestamp(columnIndex); - } - } - - public InputStream getAsciiStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /* - * (non-Javadoc) - * - * @see java.sql.ResultSet#getUnicodeStream(int) - * - * * @deprecated use getCharacterStream in place of - * getUnicodeStream - */ - @Deprecated - public InputStream getUnicodeStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public InputStream getBinaryStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getString(String columnLabel) throws SQLException { - return this.getString(this.findColumn(columnLabel)); - } - - public boolean getBoolean(String columnLabel) throws SQLException { - return this.getBoolean(this.findColumn(columnLabel)); - } - - public byte getByte(String columnLabel) throws SQLException { - return this.getByte(this.findColumn(columnLabel)); - } - - public short getShort(String columnLabel) throws SQLException { - return this.getShort(this.findColumn(columnLabel)); - } - - public int getInt(String columnLabel) throws SQLException { - return this.getInt(this.findColumn(columnLabel)); - } - - public long getLong(String columnLabel) throws SQLException { - return this.getLong(this.findColumn(columnLabel)); - } - - public float getFloat(String columnLabel) throws SQLException { - return this.getFloat(this.findColumn(columnLabel)); - } - - public double getDouble(String columnLabel) throws SQLException { - return this.getDouble(this.findColumn(columnLabel)); - } - - /* - * used by spark - */ - @Deprecated - public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { - return this.getBigDecimal(this.findColumn(columnLabel), scale); - } - - public byte[] getBytes(String columnLabel) throws SQLException { - return this.getBytes(this.findColumn(columnLabel)); - } - - public Date getDate(String columnLabel) throws SQLException { - return this.getDate(this.findColumn(columnLabel)); - } - - public Time getTime(String columnLabel) throws SQLException { - return this.getTime(this.findColumn(columnLabel)); - } - - public Timestamp getTimestamp(String columnLabel) throws SQLException { - return this.getTimestamp(this.findColumn(columnLabel)); - } - - public InputStream getAsciiStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Deprecated - public InputStream getUnicodeStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public InputStream getBinaryStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getCursorName() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSetMetaData getMetaData() throws SQLException { - return new TSDBResultSetMetaData(this.columnMetaDataList); - } - - public Object getObject(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - return this.rowData.get(colIndex); - } else { - return this.blockData.get(colIndex); - } - } - - public Object getObject(String columnLabel) throws SQLException { - return this.getObject(this.findColumn(columnLabel)); - } - - public int findColumn(String columnLabel) throws SQLException { - Iterator colMetaDataIt = this.columnMetaDataList.iterator(); - while (colMetaDataIt.hasNext()) { - ColumnMetaData colMetaData = colMetaDataIt.next(); - if (colMetaData.getColName() != null && colMetaData.getColName().equalsIgnoreCase(columnLabel)) { - return colMetaData.getColIndex() + 1; - } - } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - public Reader getCharacterStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Reader getCharacterStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /* - * used by spark - */ - public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - - if (!this.getBatchFetch()) { - this.lastWasNull = this.rowData.wasNull(colIndex); - return new BigDecimal(this.rowData.getLong(colIndex, this.columnMetaDataList.get(colIndex).getColType())); - } else { - return new BigDecimal(this.blockData.getLong(colIndex)); - } - } - - public BigDecimal getBigDecimal(String columnLabel) throws SQLException { - return this.getBigDecimal(this.findColumn(columnLabel)); - } - - public boolean isBeforeFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isAfterLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void beforeFirst() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void afterLast() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean first() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean last() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean absolute(int row) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean relative(int rows) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean previous() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setFetchDirection(int direction) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getFetchDirection() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setFetchSize(int rows) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getFetchSize() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getType() throws SQLException { - return ResultSet.TYPE_FORWARD_ONLY; - } - - public int getConcurrency() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowUpdated() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowInserted() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean rowDeleted() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNull(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBoolean(int columnIndex, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateByte(int columnIndex, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateShort(int columnIndex, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateInt(int columnIndex, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateLong(int columnIndex, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateFloat(int columnIndex, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateDouble(int columnIndex, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateString(int columnIndex, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBytes(int columnIndex, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateDate(int columnIndex, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateTime(int columnIndex, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateObject(int columnIndex, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNull(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBoolean(String columnLabel, boolean x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateByte(String columnLabel, byte x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateShort(String columnLabel, short x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateInt(String columnLabel, int x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateLong(String columnLabel, long x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateFloat(String columnLabel, float x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateDouble(String columnLabel, double x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateString(String columnLabel, String x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBytes(String columnLabel, byte[] x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateDate(String columnLabel, Date x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateTime(String columnLabel, Time x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateObject(String columnLabel, Object x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void insertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void deleteRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void refreshRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void cancelRowUpdates() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void moveToInsertRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void moveToCurrentRow() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Statement getStatement() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Object getObject(int columnIndex, Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Ref getRef(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Blob getBlob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Clob getClob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Array getArray(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Object getObject(String columnLabel, Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Ref getRef(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Blob getBlob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Clob getClob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Array getArray(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Date getDate(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Date getDate(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Time getTime(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Time getTime(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public URL getURL(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public URL getURL(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRef(int columnIndex, Ref x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRef(String columnLabel, Ref x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(int columnIndex, Blob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(String columnLabel, Blob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(int columnIndex, Clob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(String columnLabel, Clob x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateArray(int columnIndex, Array x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateArray(String columnLabel, Array x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public RowId getRowId(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public RowId getRowId(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRowId(int columnIndex, RowId x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateRowId(String columnLabel, RowId x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getHoldability() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isClosed() throws SQLException { - boolean isClosed = true; - if (jniConnector != null) { - isClosed = jniConnector.isResultsetClosed(); - } - return isClosed; - } - - public void updateNString(int columnIndex, String nString) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNString(String columnLabel, String nString) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(int columnIndex, NClob nClob) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(String columnLabel, NClob nClob) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob getNClob(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob getNClob(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML getSQLXML(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML getSQLXML(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getNString(int columnIndex) throws SQLException { - int colIndex = getTrueColumnIndex(columnIndex); - return (String) rowData.get(colIndex); - } - - public String getNString(String columnLabel) throws SQLException { - return (String) this.getString(columnLabel); - } - - public Reader getNCharacterStream(int columnIndex) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Reader getNCharacterStream(String columnLabel) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(int columnIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateClob(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(int columnIndex, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void updateNClob(String columnLabel, Reader reader) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } +public class TSDBResultSet extends AbstractResultSet implements ResultSet { + private final TSDBJNIConnector jniConnector; + private final TSDBStatement statement; + private final long resultSetPointer; + private List columnMetaDataList = new ArrayList<>(); + private final TSDBResultSetRowData rowData; + private final TSDBResultSetBlockData blockData; + + private boolean batchFetch; + private boolean lastWasNull; + private boolean isClosed; + + public void setBatchFetch(boolean batchFetch) { + this.batchFetch = batchFetch; + } + + public Boolean getBatchFetch() { + return this.batchFetch; + } + + public void setColumnMetaDataList(List columnMetaDataList) { + this.columnMetaDataList = columnMetaDataList; + } + + public TSDBResultSetRowData getRowData() { + return rowData; + } + + public TSDBResultSet(TSDBStatement statement, TSDBJNIConnector connector, long resultSetPointer) throws SQLException { + this.statement = statement; + this.jniConnector = connector; + this.resultSetPointer = resultSetPointer; + + int code = this.jniConnector.getSchemaMetaData(this.resultSetPointer, this.columnMetaDataList); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } + if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL); + } + if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0); + } + this.rowData = new TSDBResultSetRowData(this.columnMetaDataList.size()); + this.blockData = new TSDBResultSetBlockData(this.columnMetaDataList, this.columnMetaDataList.size()); + } + + public boolean next() throws SQLException { + if (this.getBatchFetch()) { + if (this.blockData.forward()) { + return true; + } + + int code = this.jniConnector.fetchBlock(this.resultSetPointer, this.blockData); + this.blockData.reset(); + + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL); + } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0); + } else return code != TSDBConstants.JNI_FETCH_END; + } else { + if (rowData != null) { + this.rowData.clear(); + } + int code = this.jniConnector.fetchRow(this.resultSetPointer, this.rowData); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL); + } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0); + } else if (code == TSDBConstants.JNI_FETCH_END) { + return false; + } else { + return true; + } + } + } + + public void close() throws SQLException { + if (isClosed) + return; + if (this.jniConnector != null) { + int code = this.jniConnector.freeResultSet(this.resultSetPointer); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL); + } + } + isClosed = true; + } + + public boolean wasNull() throws SQLException { + return this.lastWasNull; + } + + public String getString(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + String res = null; + if (this.getBatchFetch()) + return this.blockData.getString(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + res = this.rowData.getString(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + } + return res; + } + + public boolean getBoolean(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + boolean res = false; + if (this.getBatchFetch()) + return this.blockData.getBoolean(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + res = this.rowData.getBoolean(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + } + return res; + } + + public byte getByte(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + byte res = 0; + if (this.getBatchFetch()) + return (byte) this.blockData.getInt(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + res = (byte) this.rowData.getInt(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + } + return res; + } + + public short getShort(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + short res = 0; + if (this.getBatchFetch()) + return (short) this.blockData.getInt(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + res = (short) this.rowData.getInt(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + } + return res; + } + + public int getInt(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + int res = 0; + if (this.getBatchFetch()) + return this.blockData.getInt(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + res = this.rowData.getInt(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + } + return res; + } + + public long getLong(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + long res = 0L; + if (this.getBatchFetch()) + return this.blockData.getLong(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + res = this.rowData.getLong(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + } + return res; + } + + public float getFloat(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + float res = 0; + if (this.getBatchFetch()) + return (float) this.blockData.getDouble(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) + res = this.rowData.getFloat(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + + return res; + } + + public double getDouble(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + double res = 0; + if (this.getBatchFetch()) + return this.blockData.getDouble(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + res = this.rowData.getDouble(columnIndex - 1, this.columnMetaDataList.get(columnIndex - 1).getColType()); + } + return res; + } + + public byte[] getBytes(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + Object value = this.rowData.get(columnIndex - 1); + if (value == null) + return null; + + int colType = this.columnMetaDataList.get(columnIndex - 1).getColType(); + switch (colType) { + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return Longs.toByteArray((Long) value); + case TSDBConstants.TSDB_DATA_TYPE_INT: + return Ints.toByteArray((int) value); + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return Shorts.toByteArray((Short) value); + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return new byte[]{(byte) value}; + } + return value.toString().getBytes(); + } + + @Override + public Date getDate(int columnIndex) throws SQLException { + Timestamp timestamp = getTimestamp(columnIndex); + return timestamp == null ? null : new Date(timestamp.getTime()); + } + + @Override + public Time getTime(int columnIndex) throws SQLException { + Timestamp timestamp = getTimestamp(columnIndex); + return timestamp == null ? null : new Time(timestamp.getTime()); + } + + public Timestamp getTimestamp(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + Timestamp res = null; + + if (this.getBatchFetch()) + return this.blockData.getTimestamp(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + res = this.rowData.getTimestamp(columnIndex - 1); + } + return res; + } + + public ResultSetMetaData getMetaData() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + return new TSDBResultSetMetaData(this.columnMetaDataList); + } + + @Override + public Object getObject(int columnIndex) throws SQLException { + checkAvailability(columnIndex, this.columnMetaDataList.size()); + + Object res = null; + if (this.getBatchFetch()) + return this.blockData.get(columnIndex - 1); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + if (!lastWasNull) { + int colType = this.columnMetaDataList.get(columnIndex - 1).getColType(); + if (colType == TSDBConstants.TSDB_DATA_TYPE_BINARY) + res = ((String) this.rowData.get(columnIndex - 1)).getBytes(); + else + res = this.rowData.get(columnIndex - 1); + } + return res; + } + + public int findColumn(String columnLabel) throws SQLException { + for (ColumnMetaData colMetaData : this.columnMetaDataList) { + if (colMetaData.getColName() != null && colMetaData.getColName().equalsIgnoreCase(columnLabel)) { + return colMetaData.getColIndex() + 1; + } + } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_VARIABLE); + } + + @Override + public BigDecimal getBigDecimal(int columnIndex) throws SQLException { + if (this.getBatchFetch()) + return new BigDecimal(this.blockData.getLong(columnIndex - 1)); + + this.lastWasNull = this.rowData.wasNull(columnIndex - 1); + BigDecimal res = null; + if (!lastWasNull) { + int colType = this.columnMetaDataList.get(columnIndex - 1).getColType(); + switch (colType) { + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + res = new BigDecimal(Long.valueOf(this.rowData.get(columnIndex - 1).toString())); + break; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + res = new BigDecimal(Double.valueOf(this.rowData.get(columnIndex - 1).toString())); + break; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + return new BigDecimal(((Timestamp) this.rowData.get(columnIndex - 1)).getTime()); + default: + res = new BigDecimal(this.rowData.get(columnIndex - 1).toString()); + } + } + return res; + } + + @Override + public boolean isBeforeFirst() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean isAfterLast() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean isFirst() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean isLast() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void beforeFirst() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void afterLast() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean first() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean last() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public int getRow() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean absolute(int row) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean relative(int rows) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public boolean previous() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public Statement getStatement() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - private int getTrueColumnIndex(int columnIndex) throws SQLException { - if (columnIndex < this.COLUMN_INDEX_START_VALUE) { - throw new SQLException("Column Index out of range, " + columnIndex + " < " + this.COLUMN_INDEX_START_VALUE); - } - - int numOfCols = this.columnMetaDataList.size(); - if (columnIndex > numOfCols) { - throw new SQLException("Column Index out of range, " + columnIndex + " > " + numOfCols); - } + return this.statement; + } + + @Override + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + //TODO:did not use the specified timezone in cal + return getTimestamp(columnIndex); + } + + public boolean isClosed() throws SQLException { + if (isClosed) + return true; + if (jniConnector != null) { + isClosed = jniConnector.isResultsetClosed(); + } + return isClosed; + } + + public String getNString(int columnIndex) throws SQLException { + return getString(columnIndex); + } - return columnIndex - 1; - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java index 9352cf525350ff57525680f405d61c6b00c0cf55..ce5290de6616dfcc367434cc2e24899e992ddd63 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java @@ -30,468 +30,472 @@ import java.util.Collections; import java.util.List; public class TSDBResultSetBlockData { - private int numOfRows = 0; - private int rowIndex = 0; - - private List columnMetaDataList; - private ArrayList colData = null; - - public TSDBResultSetBlockData(List colMeta, int numOfCols) { - this.columnMetaDataList = colMeta; - this.colData = new ArrayList(numOfCols); - } - - public TSDBResultSetBlockData() { - this.colData = new ArrayList(); - } - - public void clear() { - int size = this.colData.size(); - if (this.colData != null) { - this.colData.clear(); - } - - setNumOfCols(size); - } - - public int getNumOfRows() { - return this.numOfRows; - } - - public void setNumOfRows(int numOfRows) { - this.numOfRows = numOfRows; - } - - public int getNumOfCols() { - return this.colData.size(); - } - - public void setNumOfCols(int numOfCols) { - this.colData = new ArrayList(numOfCols); - this.colData.addAll(Collections.nCopies(numOfCols, null)); - } - - public boolean hasMore() { - return this.rowIndex < this.numOfRows; - } - - public boolean forward() { - if (this.rowIndex > this.numOfRows) { - return false; - } - - return ((++this.rowIndex) < this.numOfRows); - } - - public void reset() { - this.rowIndex = 0; - } - - public void setBoolean(int col, boolean value) { - colData.set(col, value); - } - - public void setByteArray(int col, int length, byte[] value) { - try { - switch (this.columnMetaDataList.get(col).getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - buf.order(ByteOrder.LITTLE_ENDIAN).asCharBuffer(); - this.colData.set(col, buf); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - buf.order(ByteOrder.LITTLE_ENDIAN); - this.colData.set(col, buf); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - ShortBuffer sb = buf.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer(); - this.colData.set(col, sb); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_INT: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - IntBuffer ib = buf.order(ByteOrder.LITTLE_ENDIAN).asIntBuffer(); - this.colData.set(col, ib); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - LongBuffer lb = buf.order(ByteOrder.LITTLE_ENDIAN).asLongBuffer(); - this.colData.set(col, lb); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - FloatBuffer fb = buf.order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer(); - this.colData.set(col, fb); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - DoubleBuffer db = buf.order(ByteOrder.LITTLE_ENDIAN).asDoubleBuffer(); - this.colData.set(col, db); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_BINARY: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - buf.order(ByteOrder.LITTLE_ENDIAN); - this.colData.set(col, buf); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - LongBuffer lb = buf.order(ByteOrder.LITTLE_ENDIAN).asLongBuffer(); - this.colData.set(col, lb); - break; - } - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: { - ByteBuffer buf = ByteBuffer.wrap(value, 0, length); - buf.order(ByteOrder.LITTLE_ENDIAN); - this.colData.set(col, buf); - break; - } - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - private static class NullType { - private static final byte NULL_BOOL_VAL = 0x2; - private static final String NULL_STR = "null"; - - public String toString() { - return NullType.NULL_STR; - } - - public static boolean isBooleanNull(byte val) { - return val == NullType.NULL_BOOL_VAL; - } - - private static boolean isTinyIntNull(byte val) { - return val == Byte.MIN_VALUE; - } - - private static boolean isSmallIntNull(short val) { - return val == Short.MIN_VALUE; - } - - private static boolean isIntNull(int val) { - return val == Integer.MIN_VALUE; - } - - private static boolean isBigIntNull(long val) { - return val == Long.MIN_VALUE; - } - - private static boolean isFloatNull(float val) { - return Float.isNaN(val); - } - - private static boolean isDoubleNull(double val) { - return Double.isNaN(val); - } - - private static boolean isBinaryNull(byte[] val, int length) { - if (length != Byte.BYTES) { - return false; - } - - return val[0] == 0xFF; - } - - private static boolean isNcharNull(byte[] val, int length) { - if (length != Integer.BYTES) { - return false; - } - - return (val[0] & val[1] & val[2] & val[3]) == 0xFF; - } - - } - - /** - * The original type may not be a string type, but will be converted to by - * calling this method - * - * @param col column index - * @return - * @throws SQLException - */ - public String getString(int col) throws SQLException { - Object obj = get(col); - if (obj == null) { - return new NullType().toString(); - } - - return obj.toString(); - } - - public int getInt(int col) { - Object obj = get(col); - if (obj == null) { - return 0; - } - - int type = this.columnMetaDataList.get(col).getColType(); - switch (type) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: - case TSDBConstants.TSDB_DATA_TYPE_INT: { - return (int) obj; - } - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { - return ((Long) obj).intValue(); - } - - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { - return ((Double) obj).intValue(); - } - - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - case TSDBConstants.TSDB_DATA_TYPE_BINARY: { - return Integer.parseInt((String) obj); - } - } - - return 0; - } - - public boolean getBoolean(int col) throws SQLException { - Object obj = get(col); - if (obj == null) { - return Boolean.FALSE; - } - - int type = this.columnMetaDataList.get(col).getColType(); - switch (type) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: - case TSDBConstants.TSDB_DATA_TYPE_INT: { - return ((int) obj == 0L) ? Boolean.FALSE : Boolean.TRUE; - } - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { - return (((Long) obj) == 0L) ? Boolean.FALSE : Boolean.TRUE; - } - - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { - return (((Double) obj) == 0) ? Boolean.FALSE : Boolean.TRUE; - } - - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - case TSDBConstants.TSDB_DATA_TYPE_BINARY: { - if ("TRUE".compareToIgnoreCase((String) obj) == 0) { - return Boolean.TRUE; - } else if ("FALSE".compareToIgnoreCase((String) obj) == 0) { - return Boolean.TRUE; - } else { - throw new SQLDataException(); - } - } - } - - return Boolean.FALSE; - } - - public long getLong(int col) throws SQLException { - Object obj = get(col); - if (obj == null) { - return 0; - } - - int type = this.columnMetaDataList.get(col).getColType(); - switch (type) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: - case TSDBConstants.TSDB_DATA_TYPE_INT: { - return (int) obj; - } - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { - return (long) obj; - } - - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { - return ((Double) obj).longValue(); - } - - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - case TSDBConstants.TSDB_DATA_TYPE_BINARY: { - return Long.parseLong((String) obj); - } - } - - return 0; - } - - public Timestamp getTimestamp(int col) { - try { - return new Timestamp(getLong(col)); - } catch (SQLException e) { - e.printStackTrace(); - } - - return null; - } - - public double getDouble(int col) { - Object obj = get(col); - if (obj == null) { - return 0; - } - - int type = this.columnMetaDataList.get(col).getColType(); - switch (type) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: - case TSDBConstants.TSDB_DATA_TYPE_INT: { - return (int) obj; - } - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { - return (long) obj; - } - - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { - return (double) obj; - } - - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - case TSDBConstants.TSDB_DATA_TYPE_BINARY: { - return Double.parseDouble((String) obj); - } - } - - return 0; - } - - public Object get(int col) { - int fieldSize = this.columnMetaDataList.get(col).getColSize(); - - switch (this.columnMetaDataList.get(col).getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: { - ByteBuffer bb = (ByteBuffer) this.colData.get(col); - - byte val = bb.get(this.rowIndex); - if (NullType.isBooleanNull(val)) { - return null; - } - - return (val == 0x0) ? Boolean.FALSE : Boolean.TRUE; - } - - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: { - ByteBuffer bb = (ByteBuffer) this.colData.get(col); - - byte val = bb.get(this.rowIndex); - if (NullType.isTinyIntNull(val)) { - return null; - } - - return val; - } - - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: { - ShortBuffer sb = (ShortBuffer) this.colData.get(col); - short val = sb.get(this.rowIndex); - if (NullType.isSmallIntNull(val)) { - return null; - } - - return val; - } - - case TSDBConstants.TSDB_DATA_TYPE_INT: { - IntBuffer ib = (IntBuffer) this.colData.get(col); - int val = ib.get(this.rowIndex); - if (NullType.isIntNull(val)) { - return null; - } - - return val; - } - - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: { - LongBuffer lb = (LongBuffer) this.colData.get(col); - long val = lb.get(this.rowIndex); - if (NullType.isBigIntNull(val)) { - return null; - } - - return (long) val; - } - - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: { - FloatBuffer fb = (FloatBuffer) this.colData.get(col); - float val = fb.get(this.rowIndex); - if (NullType.isFloatNull(val)) { - return null; - } - - return val; - } - - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { - DoubleBuffer lb = (DoubleBuffer) this.colData.get(col); - double val = lb.get(this.rowIndex); - if (NullType.isDoubleNull(val)) { - return null; - } - - return val; - } - - case TSDBConstants.TSDB_DATA_TYPE_BINARY: { - ByteBuffer bb = (ByteBuffer) this.colData.get(col); - bb.position(fieldSize * this.rowIndex); - - int length = bb.getShort(); - - byte[] dest = new byte[length]; - bb.get(dest, 0, length); - if (NullType.isBinaryNull(dest, length)) { - return null; - } - - return new String(dest); - } - - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: { - ByteBuffer bb = (ByteBuffer) this.colData.get(col); - bb.position(fieldSize * this.rowIndex); - - int length = bb.getShort(); - - byte[] dest = new byte[length]; - bb.get(dest, 0, length); - if (NullType.isNcharNull(dest, length)) { - return null; - } - - try { - String ss = TaosGlobalConfig.getCharset(); - return new String(dest, ss); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - } - } - - return 0; - } + private int numOfRows = 0; + private int rowIndex = 0; + + private List columnMetaDataList; + private ArrayList colData = null; + + public TSDBResultSetBlockData(List colMeta, int numOfCols) { + this.columnMetaDataList = colMeta; + this.colData = new ArrayList(numOfCols); + } + + public TSDBResultSetBlockData() { + this.colData = new ArrayList(); + } + + public void clear() { + int size = this.colData.size(); + if (this.colData != null) { + this.colData.clear(); + } + + setNumOfCols(size); + } + + public int getNumOfRows() { + return this.numOfRows; + } + + public void setNumOfRows(int numOfRows) { + this.numOfRows = numOfRows; + } + + public int getNumOfCols() { + return this.colData.size(); + } + + public void setNumOfCols(int numOfCols) { + this.colData = new ArrayList(numOfCols); + this.colData.addAll(Collections.nCopies(numOfCols, null)); + } + + public boolean hasMore() { + return this.rowIndex < this.numOfRows; + } + + public boolean forward() { + if (this.rowIndex > this.numOfRows) { + return false; + } + + return ((++this.rowIndex) < this.numOfRows); + } + + public void reset() { + this.rowIndex = 0; + } + + public void setBoolean(int col, boolean value) { + colData.set(col, value); + } + + public void setByteArray(int col, int length, byte[] value) { + try { + switch (this.columnMetaDataList.get(col).getColType()) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + buf.order(ByteOrder.LITTLE_ENDIAN).asCharBuffer(); + this.colData.set(col, buf); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + buf.order(ByteOrder.LITTLE_ENDIAN); + this.colData.set(col, buf); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + ShortBuffer sb = buf.order(ByteOrder.LITTLE_ENDIAN).asShortBuffer(); + this.colData.set(col, sb); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_UINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + IntBuffer ib = buf.order(ByteOrder.LITTLE_ENDIAN).asIntBuffer(); + this.colData.set(col, ib); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + LongBuffer lb = buf.order(ByteOrder.LITTLE_ENDIAN).asLongBuffer(); + this.colData.set(col, lb); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + FloatBuffer fb = buf.order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer(); + this.colData.set(col, fb); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + DoubleBuffer db = buf.order(ByteOrder.LITTLE_ENDIAN).asDoubleBuffer(); + this.colData.set(col, db); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_BINARY: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + buf.order(ByteOrder.LITTLE_ENDIAN); + this.colData.set(col, buf); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + LongBuffer lb = buf.order(ByteOrder.LITTLE_ENDIAN).asLongBuffer(); + this.colData.set(col, lb); + break; + } + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: { + ByteBuffer buf = ByteBuffer.wrap(value, 0, length); + buf.order(ByteOrder.LITTLE_ENDIAN); + this.colData.set(col, buf); + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static class NullType { + private static final byte NULL_BOOL_VAL = 0x2; + private static final String NULL_STR = "null"; + + public String toString() { + return NullType.NULL_STR; + } + + public static boolean isBooleanNull(byte val) { + return val == NullType.NULL_BOOL_VAL; + } + + private static boolean isTinyIntNull(byte val) { + return val == Byte.MIN_VALUE; + } + + private static boolean isSmallIntNull(short val) { + return val == Short.MIN_VALUE; + } + + private static boolean isIntNull(int val) { + return val == Integer.MIN_VALUE; + } + + private static boolean isBigIntNull(long val) { + return val == Long.MIN_VALUE; + } + + private static boolean isFloatNull(float val) { + return Float.isNaN(val); + } + + private static boolean isDoubleNull(double val) { + return Double.isNaN(val); + } + + private static boolean isBinaryNull(byte[] val, int length) { + if (length != Byte.BYTES) { + return false; + } + + return val[0] == 0xFF; + } + + private static boolean isNcharNull(byte[] val, int length) { + if (length != Integer.BYTES) { + return false; + } + + return (val[0] & val[1] & val[2] & val[3]) == 0xFF; + } + + } + + /** + * The original type may not be a string type, but will be converted to by + * calling this method + * + * @param col column index + * @return + * @throws SQLException + */ + public String getString(int col) throws SQLException { + Object obj = get(col); + if (obj == null) { + return new NullType().toString(); + } + + return obj.toString(); + } + + public int getInt(int col) { + Object obj = get(col); + if (obj == null) { + return 0; + } + + int type = this.columnMetaDataList.get(col).getColType(); + switch (type) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: { + return (int) obj; + } + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { + return ((Long) obj).intValue(); + } + + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { + return ((Double) obj).intValue(); + } + + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: { + return Integer.parseInt((String) obj); + } + } + + return 0; + } + + public boolean getBoolean(int col) throws SQLException { + Object obj = get(col); + if (obj == null) { + return Boolean.FALSE; + } + + int type = this.columnMetaDataList.get(col).getColType(); + switch (type) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: { + return ((int) obj == 0L) ? Boolean.FALSE : Boolean.TRUE; + } + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { + return (((Long) obj) == 0L) ? Boolean.FALSE : Boolean.TRUE; + } + + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { + return (((Double) obj) == 0) ? Boolean.FALSE : Boolean.TRUE; + } + + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: { + if ("TRUE".compareToIgnoreCase((String) obj) == 0) { + return Boolean.TRUE; + } else if ("FALSE".compareToIgnoreCase((String) obj) == 0) { + return Boolean.TRUE; + } else { + throw new SQLDataException(); + } + } + } + + return Boolean.FALSE; + } + + public long getLong(int col) throws SQLException { + Object obj = get(col); + if (obj == null) { + return 0; + } + + int type = this.columnMetaDataList.get(col).getColType(); + switch (type) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: { + return (int) obj; + } + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { + return (long) obj; + } + + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { + return ((Double) obj).longValue(); + } + + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: { + return Long.parseLong((String) obj); + } + } + + return 0; + } + + public Timestamp getTimestamp(int col) { + try { + return new Timestamp(getLong(col)); + } catch (SQLException e) { + e.printStackTrace(); + } + + return null; + } + + public double getDouble(int col) { + Object obj = get(col); + if (obj == null) { + return 0; + } + + int type = this.columnMetaDataList.get(col).getColType(); + switch (type) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: { + return (int) obj; + } + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: { + return (long) obj; + } + + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { + return (double) obj; + } + + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: { + return Double.parseDouble((String) obj); + } + } + + return 0; + } + + public Object get(int col) { + int fieldSize = this.columnMetaDataList.get(col).getColSize(); + + switch (this.columnMetaDataList.get(col).getColType()) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: { + ByteBuffer bb = (ByteBuffer) this.colData.get(col); + + byte val = bb.get(this.rowIndex); + if (NullType.isBooleanNull(val)) { + return null; + } + + return (val == 0x0) ? Boolean.FALSE : Boolean.TRUE; + } + + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: { + ByteBuffer bb = (ByteBuffer) this.colData.get(col); + + byte val = bb.get(this.rowIndex); + if (NullType.isTinyIntNull(val)) { + return null; + } + + return val; + } + + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: { + ShortBuffer sb = (ShortBuffer) this.colData.get(col); + short val = sb.get(this.rowIndex); + if (NullType.isSmallIntNull(val)) { + return null; + } + + return val; + } + + case TSDBConstants.TSDB_DATA_TYPE_INT: { + IntBuffer ib = (IntBuffer) this.colData.get(col); + int val = ib.get(this.rowIndex); + if (NullType.isIntNull(val)) { + return null; + } + + return val; + } + + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: { + LongBuffer lb = (LongBuffer) this.colData.get(col); + long val = lb.get(this.rowIndex); + if (NullType.isBigIntNull(val)) { + return null; + } + + return (long) val; + } + + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: { + FloatBuffer fb = (FloatBuffer) this.colData.get(col); + float val = fb.get(this.rowIndex); + if (NullType.isFloatNull(val)) { + return null; + } + + return val; + } + + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: { + DoubleBuffer lb = (DoubleBuffer) this.colData.get(col); + double val = lb.get(this.rowIndex); + if (NullType.isDoubleNull(val)) { + return null; + } + + return val; + } + + case TSDBConstants.TSDB_DATA_TYPE_BINARY: { + ByteBuffer bb = (ByteBuffer) this.colData.get(col); + bb.position(fieldSize * this.rowIndex); + + int length = bb.getShort(); + + byte[] dest = new byte[length]; + bb.get(dest, 0, length); + if (NullType.isBinaryNull(dest, length)) { + return null; + } + + return new String(dest); + } + + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: { + ByteBuffer bb = (ByteBuffer) this.colData.get(col); + bb.position(fieldSize * this.rowIndex); + + int length = bb.getShort(); + + byte[] dest = new byte[length]; + bb.get(dest, 0, length); + if (NullType.isNcharNull(dest, length)) { + return null; + } + + try { + String ss = TaosGlobalConfig.getCharset(); + return new String(dest, ss); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + } + + return 0; + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java index d6d69bd8b02db7de9350c023e696f4cf2e2f5e65..e4ac5303d09f132279e29039bf6b5b812b6c5404 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetMetaData.java @@ -20,168 +20,162 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.List; -public class TSDBResultSetMetaData implements ResultSetMetaData { - - List colMetaDataList = null; - - public TSDBResultSetMetaData(List metaDataList) { - this.colMetaDataList = metaDataList; - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getColumnCount() throws SQLException { - return colMetaDataList.size(); - } - - public boolean isAutoIncrement(int column) throws SQLException { - return false; - } - - public boolean isCaseSensitive(int column) throws SQLException { - return false; - } - - public boolean isSearchable(int column) throws SQLException { - if (column == 1) { - return true; - } - return false; - } - - public boolean isCurrency(int column) throws SQLException { - return false; - } - - public int isNullable(int column) throws SQLException { - if (column == 1) { - return columnNoNulls; - } - return columnNullable; - } - - public boolean isSigned(int column) throws SQLException { - ColumnMetaData meta = this.colMetaDataList.get(column - 1); - switch (meta.getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: - case TSDBConstants.TSDB_DATA_TYPE_INT: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: - return true; - default: - return false; - } - } - - public int getColumnDisplaySize(int column) throws SQLException { - return colMetaDataList.get(column - 1).getColSize(); - } - - public String getColumnLabel(int column) throws SQLException { - return colMetaDataList.get(column - 1).getColName(); - } - - public String getColumnName(int column) throws SQLException { - return colMetaDataList.get(column - 1).getColName(); - } - - public String getSchemaName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getPrecision(int column) throws SQLException { - ColumnMetaData columnMetaData = this.colMetaDataList.get(column - 1); - switch (columnMetaData.getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - return 5; - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: - return 9; - case TSDBConstants.TSDB_DATA_TYPE_BINARY: - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - return columnMetaData.getColSize(); - default: - return 0; - } - } - - public int getScale(int column) throws SQLException { - ColumnMetaData meta = this.colMetaDataList.get(column - 1); - switch (meta.getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - return 5; - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: - return 9; - default: - return 0; - } - } - - public String getTableName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getCatalogName(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getColumnType(int column) throws SQLException { - ColumnMetaData meta = this.colMetaDataList.get(column - 1); - switch (meta.getColType()) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: - return java.sql.Types.BIT; - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: - return java.sql.Types.TINYINT; - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: - return java.sql.Types.SMALLINT; - case TSDBConstants.TSDB_DATA_TYPE_INT: - return java.sql.Types.INTEGER; - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: - return java.sql.Types.BIGINT; - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: - return java.sql.Types.FLOAT; - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: - return java.sql.Types.DOUBLE; - case TSDBConstants.TSDB_DATA_TYPE_BINARY: - return java.sql.Types.CHAR; - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - return java.sql.Types.BIGINT; - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - return java.sql.Types.CHAR; - } - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - public String getColumnTypeName(int column) throws SQLException { - ColumnMetaData meta = this.colMetaDataList.get(column - 1); - return TSDBConstants.DATATYPE_MAP.get(meta.getColType()); - } - - public boolean isReadOnly(int column) throws SQLException { - return true; - } - - public boolean isWritable(int column) throws SQLException { - return false; - } - - public boolean isDefinitelyWritable(int column) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getColumnClassName(int column) throws SQLException { - int columnType = getColumnType(column); - String columnClassName = ""; - switch (columnType) { - case Types.TIMESTAMP: - columnClassName = Timestamp.class.getName(); +public class TSDBResultSetMetaData extends WrapperImpl implements ResultSetMetaData { + + List colMetaDataList; + + public TSDBResultSetMetaData(List metaDataList) { + this.colMetaDataList = metaDataList; + } + + public int getColumnCount() throws SQLException { + return colMetaDataList.size(); + } + + public boolean isAutoIncrement(int column) throws SQLException { + return false; + } + + public boolean isCaseSensitive(int column) throws SQLException { + return false; + } + + public boolean isSearchable(int column) throws SQLException { + if (column == 1) { + return true; + } + return false; + } + + public boolean isCurrency(int column) throws SQLException { + return false; + } + + public int isNullable(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + if (column == 1) { + return columnNoNulls; + } + return columnNullable; + } + + public boolean isSigned(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + ColumnMetaData meta = this.colMetaDataList.get(column - 1); + switch (meta.getColType()) { + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + case TSDBConstants.TSDB_DATA_TYPE_INT: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return true; + default: + return false; + } + } + + public int getColumnDisplaySize(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + return colMetaDataList.get(column - 1).getColSize(); + } + + public String getColumnLabel(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + return colMetaDataList.get(column - 1).getColName(); + } + + public String getColumnName(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + return colMetaDataList.get(column - 1).getColName(); + } + + public String getSchemaName(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public int getPrecision(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + ColumnMetaData columnMetaData = this.colMetaDataList.get(column - 1); + switch (columnMetaData.getColType()) { + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return 5; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return 9; + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + return columnMetaData.getColSize(); + default: + return 0; + } + } + + public int getScale(int column) throws SQLException { + if (column < 1 && column >= colMetaDataList.size()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + ColumnMetaData meta = this.colMetaDataList.get(column - 1); + switch (meta.getColType()) { + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return 5; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return 9; + default: + return 0; + } + } + + public String getTableName(int column) throws SQLException { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public String getCatalogName(int column) throws SQLException { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public int getColumnType(int column) throws SQLException { + ColumnMetaData meta = this.colMetaDataList.get(column - 1); + return TSDBConstants.taosType2JdbcType(meta.getColType()); + } + + public String getColumnTypeName(int column) throws SQLException { + ColumnMetaData meta = this.colMetaDataList.get(column - 1); + return TSDBConstants.taosType2JdbcTypeName(meta.getColType()); + } + + public boolean isReadOnly(int column) throws SQLException { + return true; + } + + public boolean isWritable(int column) throws SQLException { + return false; + } + + public boolean isDefinitelyWritable(int column) throws SQLException { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + public String getColumnClassName(int column) throws SQLException { + int columnType = getColumnType(column); + String columnClassName = ""; + switch (columnType) { + case Types.TIMESTAMP: + columnClassName = Timestamp.class.getName(); break; case Types.CHAR: columnClassName = String.class.getName(); @@ -198,8 +192,8 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { case Types.INTEGER: columnClassName = Integer.class.getName(); break; - case Types.SMALLINT: - columnClassName = Short.class.getName(); + case Types.SMALLINT: + columnClassName = Short.class.getName(); break; case Types.TINYINT: columnClassName = Byte.class.getName(); @@ -207,7 +201,7 @@ public class TSDBResultSetMetaData implements ResultSetMetaData { case Types.BIT: columnClassName = Boolean.class.getName(); break; - } + } return columnClassName; - } + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java index c57f19550dd14719baecb835d76263df1e6a669b..7cf5f0d79a89d0e6ed21d6b18136b92dc2c0b131 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java @@ -14,208 +14,316 @@ *****************************************************************************/ package com.taosdata.jdbc; +import java.math.BigDecimal; import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collections; public class TSDBResultSetRowData { - private ArrayList data = null; - private int colSize = 0; - - public TSDBResultSetRowData(int colSize) { - this.setColSize(colSize); - } - - public TSDBResultSetRowData() { - this.data = new ArrayList(); - this.setColSize(0); - } - - public void clear() { - if(this.data != null) { - this.data.clear(); - } - if (this.colSize == 0) { - return; - } - this.data = new ArrayList(colSize); - this.data.addAll(Collections.nCopies(this.colSize, null)); - } - - public boolean wasNull(int col) { - return data.get(col) == null; - } - - public void setBoolean(int col, boolean value) { - data.set(col, value); - } - - public boolean getBoolean(int col, int srcType) throws SQLException { - Object obj = data.get(col); - - switch(srcType) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: return (Boolean) obj; - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return ((Float) obj) == 1.0? Boolean.TRUE:Boolean.FALSE; - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return ((Double) obj) == 1.0? Boolean.TRUE:Boolean.FALSE; - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return ((Byte) obj) == 1? Boolean.TRUE:Boolean.FALSE; - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT:return ((Short)obj) == 1? Boolean.TRUE:Boolean.FALSE; - case TSDBConstants.TSDB_DATA_TYPE_INT: return ((Integer)obj) == 1? Boolean.TRUE:Boolean.FALSE; - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return ((Long) obj) == 1L? Boolean.TRUE:Boolean.FALSE; - } - - return Boolean.TRUE; - } - - public void setByte(int col, byte value) { - data.set(col, value); - } - - public void setShort(int col, short value) { - data.set(col, value); - } - - public void setInt(int col, int value) { - data.set(col, value); - } - - public int getInt(int col, int srcType) throws SQLException { - Object obj = data.get(col); - - switch(srcType) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return ((Float) obj).intValue(); - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return ((Double)obj).intValue(); - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return (Byte) obj; - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT:return (Short) obj; - case TSDBConstants.TSDB_DATA_TYPE_INT: return (Integer) obj; - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return ((Long) obj).intValue(); - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - case TSDBConstants.TSDB_DATA_TYPE_BINARY: return Integer.parseInt((String) obj); - } - - return 0; - } - - public void setLong(int col, long value) { - data.set(col, value); - } - - public long getLong(int col, int srcType) throws SQLException { - Object obj = data.get(col); - - switch(srcType) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return ((Float) obj).longValue(); - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return ((Double) obj).longValue(); - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return (Byte) obj; - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT:return (Short) obj; - case TSDBConstants.TSDB_DATA_TYPE_INT: return (Integer) obj; - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return (Long) obj; - case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - case TSDBConstants.TSDB_DATA_TYPE_BINARY: return Long.parseLong((String) obj); - } - - return 0; - } - - public void setFloat(int col, float value) { - data.set(col, value); - } - - public float getFloat(int col, int srcType) throws SQLException { - Object obj = data.get(col); - - switch(srcType) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return (Float) obj; - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return ((Double) obj).floatValue(); - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return (Byte) obj; - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: return (Short) obj; - case TSDBConstants.TSDB_DATA_TYPE_INT: return (Integer) obj; - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return (Long) obj; - } - - return 0; - } - - public void setDouble(int col, double value) { - data.set(col, value); - } - - public double getDouble(int col, int srcType) throws SQLException { - Object obj = data.get(col); - - switch(srcType) { - case TSDBConstants.TSDB_DATA_TYPE_BOOL: return Boolean.TRUE.equals(obj)? 1:0; - case TSDBConstants.TSDB_DATA_TYPE_FLOAT: return (Float) obj; - case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: return (Double) obj; - case TSDBConstants.TSDB_DATA_TYPE_TINYINT: return (Byte) obj; - case TSDBConstants.TSDB_DATA_TYPE_SMALLINT:return (Short) obj; - case TSDBConstants.TSDB_DATA_TYPE_INT: return (Integer) obj; - case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: - case TSDBConstants.TSDB_DATA_TYPE_BIGINT: return (Long) obj; - } - - return 0; - } - - public void setString(int col, String value) { - data.set(col, value); - } - - public void setByteArray(int col, byte[] value) { + private ArrayList data; + private int colSize = 0; + + public TSDBResultSetRowData(int colSize) { + this.setColSize(colSize); + } + + public void clear() { + if (this.data != null) { + this.data.clear(); + } + if (this.colSize == 0) { + return; + } + this.data = new ArrayList<>(colSize); + this.data.addAll(Collections.nCopies(this.colSize, null)); + } + + public boolean wasNull(int col) { + return data.get(col) == null; + } + + public void setBoolean(int col, boolean value) { + data.set(col, value); + } + + public boolean getBoolean(int col, int srcType) throws SQLException { + Object obj = data.get(col); + + switch (srcType) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return (Boolean) obj; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return ((Float) obj) == 1.0 ? Boolean.TRUE : Boolean.FALSE; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return ((Double) obj) == 1.0 ? Boolean.TRUE : Boolean.FALSE; + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return ((Byte) obj) == 1 ? Boolean.TRUE : Boolean.FALSE; + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return ((Short) obj) == 1 ? Boolean.TRUE : Boolean.FALSE; + case TSDBConstants.TSDB_DATA_TYPE_INT: + return ((Integer) obj) == 1 ? Boolean.TRUE : Boolean.FALSE; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return ((Long) obj) == 1L ? Boolean.TRUE : Boolean.FALSE; + default: + return false; + } + } + + public void setByte(int col, byte value) { + data.set(col, value); + } + + public void setShort(int col, short value) { + data.set(col, value); + } + + public void setInt(int col, int value) { + data.set(col, value); + } + + public int getInt(int col, int srcType) throws SQLException { + Object obj = data.get(col); + + switch (srcType) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return Boolean.TRUE.equals(obj) ? 1 : 0; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return ((Float) obj).intValue(); + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return ((Double) obj).intValue(); + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return (Byte) obj; + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return (Short) obj; + case TSDBConstants.TSDB_DATA_TYPE_INT: + return (Integer) obj; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return ((Long) obj).intValue(); + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + return Integer.parseInt((String) obj); + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: { + Byte value = (byte) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: { + short value = (short) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + case TSDBConstants.TSDB_DATA_TYPE_UINT: { + int value = (int) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: { + long value = (long) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return new Long(value).intValue(); + } + } + + return 0; + } + + public void setLong(int col, long value) { + data.set(col, value); + } + + public long getLong(int col, int srcType) throws SQLException { + Object obj = data.get(col); + + switch (srcType) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return Boolean.TRUE.equals(obj) ? 1 : 0; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return ((Float) obj).longValue(); + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return ((Double) obj).longValue(); + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return (Byte) obj; + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return (Short) obj; + case TSDBConstants.TSDB_DATA_TYPE_INT: + return (Integer) obj; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return (Long) obj; + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + return Long.parseLong((String) obj); + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: { + Byte value = (byte) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: { + short value = (short) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + case TSDBConstants.TSDB_DATA_TYPE_UINT: { + int value = (int) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: { + long value = (long) obj; + if (value < 0) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE); + return value; + } + } + + return 0; + } + + public void setFloat(int col, float value) { + data.set(col, value); + } + + public float getFloat(int col, int srcType) { + Object obj = data.get(col); + + switch (srcType) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return Boolean.TRUE.equals(obj) ? 1 : 0; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return (Float) obj; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return ((Double) obj).floatValue(); + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return (Byte) obj; + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return (Short) obj; + case TSDBConstants.TSDB_DATA_TYPE_INT: + return (Integer) obj; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return (Long) obj; + } + + return 0; + } + + public void setDouble(int col, double value) { + data.set(col, value); + } + + public double getDouble(int col, int srcType) { + Object obj = data.get(col); + + switch (srcType) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return Boolean.TRUE.equals(obj) ? 1 : 0; + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return (Float) obj; + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return (Double) obj; + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return (Byte) obj; + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return (Short) obj; + case TSDBConstants.TSDB_DATA_TYPE_INT: + return (Integer) obj; + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return (Long) obj; + } + + return 0; + } + + public void setString(int col, String value) { + data.set(col, value); + } + + public void setByteArray(int col, byte[] value) { try { data.set(col, new String(value, TaosGlobalConfig.getCharset())); } catch (Exception e) { e.printStackTrace(); } - } - - /** - * The original type may not be a string type, but will be converted to by calling this method - * @param col column index - * @return - * @throws SQLException - */ - public String getString(int col, int srcType) throws SQLException { - if (srcType == TSDBConstants.TSDB_DATA_TYPE_BINARY || srcType == TSDBConstants.TSDB_DATA_TYPE_NCHAR) { - return (String) data.get(col); - } else { - return String.valueOf(data.get(col)); - } - } - - public void setTimestamp(int col, long ts) { - data.set(col, ts); - } - - public Timestamp getTimestamp(int col) { - return new Timestamp((Long) data.get(col)); - } - - public Object get(int col) { - return data.get(col); - } - - public int getColSize() { - return colSize; - } - - public void setColSize(int colSize) { - this.colSize = colSize; - this.clear(); - } - - public ArrayList getData() { - return data; - } - - public void setData(ArrayList data) { - this.data = (ArrayList) data.clone(); - } + } + + /** + * The original type may not be a string type, but will be converted to by calling this method + * + * @param col column index + * @return + */ + public String getString(int col, int srcType) { + switch (srcType) { + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + return (String) data.get(col); + case TSDBConstants.TSDB_DATA_TYPE_UTINYINT: { + Byte value = new Byte(String.valueOf(data.get(col))); + if (value >= 0) + return value.toString(); + return Integer.toString(value & 0xff); + } + case TSDBConstants.TSDB_DATA_TYPE_USMALLINT: { + Short value = new Short(String.valueOf(data.get(col))); + if (value >= 0) + return value.toString(); + return Integer.toString(value & 0xffff); + } + case TSDBConstants.TSDB_DATA_TYPE_UINT: { + Integer value = new Integer(String.valueOf(data.get(col))); + if (value >= 0) + return value.toString(); + return Long.toString(value & 0xffffffffl); + } + case TSDBConstants.TSDB_DATA_TYPE_UBIGINT: { + Long value = new Long(String.valueOf(data.get(col))); + if (value >= 0) + return value.toString(); + long lowValue = value & 0x7fffffffffffffffL; + return BigDecimal.valueOf(lowValue).add(BigDecimal.valueOf(Long.MAX_VALUE)).add(BigDecimal.valueOf(1)).toString(); + } + default: + return String.valueOf(data.get(col)); + } + } + + public void setTimestamp(int col, long ts) { + data.set(col, new Timestamp(ts)); + } + + public Timestamp getTimestamp(int col) { + return (Timestamp) data.get(col); + } + + public Object get(int col) { + return data.get(col); + } + + public int getColSize() { + return colSize; + } + + private void setColSize(int colSize) { + this.colSize = colSize; + this.clear(); + } + + public ArrayList getData() { + return data; + } + + public void setData(ArrayList data) { + this.data = (ArrayList) data.clone(); + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java index 059962a7a120073c17258652f9fb3609cc5072be..48854e773f89a45784de3cd709ec5bbe6185e09b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetWrapper.java @@ -1153,11 +1153,11 @@ public class TSDBResultSetWrapper implements ResultSet { } public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index cd2a768a38c377916004875f3f6f143b555a0294..fb20a621b0f11413e13e4234be357f6456a2a4fa 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -14,32 +14,20 @@ *****************************************************************************/ package com.taosdata.jdbc; -import java.sql.*; -import java.util.ArrayList; -import java.util.List; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; -public class TSDBStatement implements Statement { - private TSDBJNIConnector connector = null; - - /** - * To store batched commands - */ - protected List batchedArgs; - - /** - * Timeout for a query - */ - protected int queryTimeout = 0; - - private Long pSql = 0l; +public class TSDBStatement extends AbstractStatement { + private TSDBJNIConnector connector; /** * Status of current statement */ - private boolean isClosed = true; - private int affectedRows = 0; - + private boolean isClosed; + private int affectedRows = -1; private TSDBConnection connection; + private TSDBResultSet resultSet; public void setConnection(TSDBConnection connection) { this.connection = connection; @@ -48,287 +36,95 @@ public class TSDBStatement implements Statement { TSDBStatement(TSDBConnection connection, TSDBJNIConnector connector) { this.connection = connection; this.connector = connector; - this.isClosed = false; - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); } public ResultSet executeQuery(String sql) throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - - // TODO make sure it is not a update query - pSql = this.connector.executeQuery(sql); - - long resultSetPointer = this.connector.getResultSet(); - - if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { - this.connector.freeResultSet(pSql); - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - - // create/insert/update/delete/alter - if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { - this.connector.freeResultSet(pSql); - return null; - } - - if (!this.connector.isUpdateQuery(pSql)) { - TSDBResultSet res = new TSDBResultSet(this.connector, resultSetPointer); - res.setBatchFetch(this.connection.getBatchFetch()); - return res; - } else { + // check if closed + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + //TODO: 如果在executeQuery方法中执行insert语句,那么先执行了SQL,再通过pSql来检查是否为一个insert语句,但这个insert SQL已经执行成功了 + + // execute query + long pSql = this.connector.executeQuery(sql); + // if pSql is create/insert/update/delete/alter SQL + if (this.connector.isUpdateQuery(pSql)) { this.connector.freeResultSet(pSql); - return null; + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY); } - + TSDBResultSet res = new TSDBResultSet(this, this.connector, pSql); + res.setBatchFetch(this.connection.getBatchFetch()); + return res; } public int executeUpdate(String sql) throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - // TODO check if current query is update query - pSql = this.connector.executeQuery(sql); - long resultSetPointer = this.connector.getResultSet(); - - if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { + long pSql = this.connector.executeQuery(sql); + // if pSql is create/insert/update/delete/alter SQL + if (!this.connector.isUpdateQuery(pSql)) { this.connector.freeResultSet(pSql); - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEUPDATE); } - - this.affectedRows = this.connector.getAffectedRows(pSql); + int affectedRows = this.connector.getAffectedRows(pSql); this.connector.freeResultSet(pSql); - - return this.affectedRows; - } - - public String getErrorMsg(long pSql) { - return this.connector.getErrMsg(pSql); + return affectedRows; } public void close() throws SQLException { - if (!isClosed) { - if (!this.connector.isResultsetClosed()) { - this.connector.freeResultSet(); - } - isClosed = true; - } - } - - public int getMaxFieldSize() throws SQLException { - return 0; -// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setMaxFieldSize(int max) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getMaxRows() throws SQLException { - // always set maxRows to zero, meaning unlimitted rows in a resultSet - return 0; - } - - public void setMaxRows(int max) throws SQLException { - // always set maxRows to zero, meaning unlimited rows in a resultSet - } - - public void setEscapeProcessing(boolean enable) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getQueryTimeout() throws SQLException { - return queryTimeout; - } - - public void setQueryTimeout(int seconds) throws SQLException { - this.queryTimeout = seconds; - } - - public void cancel() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setCursorName(String name) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed) + return; + if (this.resultSet != null && !this.resultSet.isClosed()) + this.resultSet.close(); + isClosed = true; } public boolean execute(String sql) throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - boolean res = true; - pSql = this.connector.executeQuery(sql); - long resultSetPointer = this.connector.getResultSet(); - - if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { - this.connector.freeResultSet(pSql); - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { - // no result set is retrieved + // check if closed + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + // execute query + long pSql = this.connector.executeQuery(sql); + // if pSql is create/insert/update/delete/alter SQL + if (this.connector.isUpdateQuery(pSql)) { + this.affectedRows = this.connector.getAffectedRows(pSql); this.connector.freeResultSet(pSql); - res = false; + return false; } - return res; + this.resultSet = new TSDBResultSet(this, this.connector, pSql); + this.resultSet.setBatchFetch(this.connection.getBatchFetch()); + return true; } public ResultSet getResultSet() throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - long resultSetPointer = connector.getResultSet(); - TSDBResultSet resSet = null; - if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - resSet = new TSDBResultSet(connector, resultSetPointer); - } - return resSet; + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); +// long resultSetPointer = connector.getResultSet(); +// TSDBResultSet resSet = null; +// if (resultSetPointer != TSDBConstants.JNI_NULL_POINTER) { +// resSet = new TSDBResultSet(connector, resultSetPointer); +// } + return this.resultSet; } public int getUpdateCount() throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); return this.affectedRows; } - public boolean getMoreResults() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setFetchDirection(int direction) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getFetchDirection() throws SQLException { - return ResultSet.FETCH_FORWARD; -// throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /* - * used by spark - */ - public void setFetchSize(int rows) throws SQLException { - } - - /* - * used by spark - */ - public int getFetchSize() throws SQLException { - return 4096; - } - - public int getResultSetConcurrency() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getResultSetType() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void addBatch(String sql) throws SQLException { - if (batchedArgs == null) { - batchedArgs = new ArrayList<>(); - } - batchedArgs.add(sql); - } - - public void clearBatch() throws SQLException { - batchedArgs.clear(); - } - - public int[] executeBatch() throws SQLException { - if (isClosed) { - throw new SQLException("Invalid method call on a closed statement."); - } - if (batchedArgs == null) { - throw new SQLException(TSDBConstants.WrapErrMsg("Batch is empty!")); - } else { - int[] res = new int[batchedArgs.size()]; - for (int i = 0; i < batchedArgs.size(); i++) { - res[i] = executeUpdate(batchedArgs.get(i)); - } - return res; - } - } - public Connection getConnection() throws SQLException { - if (this.connector != null) - return this.connection; - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean getMoreResults(int current) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public ResultSet getGeneratedKeys() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int executeUpdate(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean execute(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean execute(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public int getResultSetHoldability() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (this.connector == null) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + return this.connection; } public boolean isClosed() throws SQLException { return isClosed; } - public void setPoolable(boolean poolable) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isPoolable() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - public void closeOnCompletion() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isCloseOnCompletion() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java index deffd9aa2ae88802f71af5cbec66c5896cf4e19a..c5fd497ca3dfed8bc8555110660ff70a9fd23447 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java @@ -14,43 +14,34 @@ *****************************************************************************/ package com.taosdata.jdbc; -import javax.management.OperationsException; import java.sql.SQLException; -import java.util.Map; -import java.util.TimerTask; -import java.util.concurrent.*; public class TSDBSubscribe { - private TSDBJNIConnector connecter = null; - private long id = 0; + private final TSDBJNIConnector connecter; + private final long id; TSDBSubscribe(TSDBJNIConnector connecter, long id) throws SQLException { - if (null != connecter) { - this.connecter = connecter; - this.id = id; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } + if (connecter == null) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + + this.connecter = connecter; + this.id = id; } /** * consume - * - * @throws OperationsException, SQLException */ - public TSDBResultSet consume() throws OperationsException, SQLException { - if (this.connecter.isClosed()) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } + public TSDBResultSet consume() throws SQLException { + if (this.connecter.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); long resultSetPointer = this.connecter.consume(this.id); - if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { return null; } else { - return new TSDBResultSet(this.connecter, resultSetPointer); + return new TSDBResultSet(null, this.connecter, resultSetPointer); } } @@ -61,9 +52,9 @@ public class TSDBSubscribe { * @throws SQLException */ public void close(boolean keepProgress) throws SQLException { - if (this.connecter.isClosed()) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } + if (this.connecter.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + this.connecter.unsubscribe(this.id, keepProgress); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..5b7539d434e0c5595d75da0a50baca9ab59c953b --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/WrapperImpl.java @@ -0,0 +1,21 @@ +package com.taosdata.jdbc; + +import java.sql.SQLException; +import java.sql.Wrapper; + +public class WrapperImpl implements Wrapper { + + @Override + public T unwrap(Class iface) throws SQLException { + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return iface.isInstance(this); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java deleted file mode 100644 index ef78292de60d11b535df7403cd27d622686fbba1..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/bean/TSDBPreparedParam.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.taosdata.jdbc.bean; - -import java.util.List; - -/** - * tdengine batch insert or import param object - */ -public class TSDBPreparedParam { - - /** - * tableName, if sTable Name is not null, and this is sub table name. - */ - private String tableName; - - /** - * sub middle param list - */ - private List middleParamList; - - /** - * value list - */ - private List valueList; - - public TSDBPreparedParam(String tableName) { - this.tableName = tableName; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public List getMiddleParamList() { - return middleParamList; - } - - public void setMiddleParamList(List middleParamList) { - this.middleParamList = middleParamList; - } - - public void setMiddleParam(int parameterIndex, Object x) { - this.middleParamList.set(parameterIndex, x); - } - - public List getValueList() { - return valueList; - } - - public void setValueList(List valueList) { - this.valueList = valueList; - } - - - public void setValueParam(int parameterIndex, Object x) { - this.valueList.set(parameterIndex, x); - } - -} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index 5260b780bd510edad6ef8ea9a481fa334cca50f6..1f3ed2d144a88b10c0b1a8947e70def2c3db42a9 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -1,34 +1,29 @@ package com.taosdata.jdbc.rs; -import com.taosdata.jdbc.TSDBConstants; +import com.taosdata.jdbc.AbstractConnection; import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; -import java.sql.*; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; import java.util.Properties; -import java.util.concurrent.Executor; -public class RestfulConnection implements Connection { +public class RestfulConnection extends AbstractConnection { - private static final String CONNECTION_IS_CLOSED = "connection is closed."; - private static final String AUTO_COMMIT_IS_TRUE = "auto commit is true"; private final String host; private final int port; - private final Properties props; - private volatile String database; private final String url; + private volatile String database; /******************************************************/ private boolean isClosed; - private DatabaseMetaData metadata; - private Map> typeMap; - private Properties clientInfoProps = new Properties(); + private final DatabaseMetaData metadata; public RestfulConnection(String host, String port, Properties props, String database, String url) { this.host = host; this.port = Integer.parseInt(port); - this.props = props; this.database = database; this.url = url; this.metadata = new RestfulDatabaseMetaData(url, props.getProperty(TSDBDriver.PROPERTY_KEY_USER), this); @@ -37,7 +32,7 @@ public class RestfulConnection implements Connection { @Override public Statement createStatement() throws SQLException { if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); return new RestfulStatement(this, database); } @@ -45,59 +40,8 @@ public class RestfulConnection implements Connection { @Override public PreparedStatement prepareStatement(String sql) throws SQLException { if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - //TODO: prepareStatement - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public CallableStatement prepareCall(String sql) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public String nativeSQL(String sql) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - - //nothing did - return sql; - } - - @Override - public void setAutoCommit(boolean autoCommit) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (!autoCommit) - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public boolean getAutoCommit() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - return true; - } - - @Override - public void commit() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (getAutoCommit()) - throw new SQLException(AUTO_COMMIT_IS_TRUE); - //nothing to do - } - - @Override - public void rollback() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (getAutoCommit()) - throw new SQLException(AUTO_COMMIT_IS_TRUE); - //nothing to do + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return new RestfulPreparedStatement(this, database, sql); } @Override @@ -116,356 +60,11 @@ public class RestfulConnection implements Connection { @Override public DatabaseMetaData getMetaData() throws SQLException { if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); return this.metadata; } - @Override - public void setReadOnly(boolean readOnly) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - // nothing to do - } - - @Override - public boolean isReadOnly() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - return true; - } - - @Override - public void setCatalog(String catalog) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - synchronized (RestfulConnection.class) { - this.database = catalog; - } - } - - @Override - public String getCatalog() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - return this.database; - } - - @Override - public void setTransactionIsolation(int level) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - switch (level) { - case Connection.TRANSACTION_NONE: - break; - case Connection.TRANSACTION_READ_UNCOMMITTED: - case Connection.TRANSACTION_READ_COMMITTED: - case Connection.TRANSACTION_REPEATABLE_READ: - case Connection.TRANSACTION_SERIALIZABLE: - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - default: - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - } - - @Override - public int getTransactionIsolation() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - //Connection.TRANSACTION_NONE specifies that transactions are not supported. - return Connection.TRANSACTION_NONE; - } - - @Override - public SQLWarning getWarnings() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - - return null; - } - - @Override - public void clearWarnings() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - //nothing to do - } - - @Override - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - - if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - return createStatement(); - } - - @Override - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (resultSetType != ResultSet.TYPE_FORWARD_ONLY || resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) - throw new SQLFeatureNotSupportedException(TSDBConstants.INVALID_VARIABLES); - - return this.prepareStatement(sql); - } - - @Override - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (resultSetType != ResultSet.TYPE_FORWARD_ONLY || resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) - throw new SQLFeatureNotSupportedException(TSDBConstants.INVALID_VARIABLES); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Map> getTypeMap() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - - synchronized (RestfulConnection.class) { - if (this.typeMap == null) { - this.typeMap = new HashMap<>(); - } - return this.typeMap; - } - } - - @Override - public void setTypeMap(Map> map) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - - synchronized (RestfulConnection.class) { - this.typeMap = map; - } - } - - @Override - public void setHoldability(int holdability) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (holdability != ResultSet.HOLD_CURSORS_OVER_COMMIT) - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public int getHoldability() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - return ResultSet.HOLD_CURSORS_OVER_COMMIT; - } - - @Override - public Savepoint setSavepoint() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (getAutoCommit()) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - //nothing to do - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Savepoint setSavepoint(String name) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (getAutoCommit()) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - //nothing to do - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void rollback(Savepoint savepoint) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - if (getAutoCommit()) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - //nothing to do - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void releaseSavepoint(Savepoint savepoint) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - if (resultSetHoldability != ResultSet.HOLD_CURSORS_OVER_COMMIT) - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - return createStatement(resultSetType, resultSetConcurrency); - } - - @Override - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - if (resultSetHoldability != ResultSet.HOLD_CURSORS_OVER_COMMIT) - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - return prepareStatement(sql, resultSetType, resultSetConcurrency); - } - - @Override - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Clob createClob() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Blob createBlob() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public NClob createNClob() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public SQLXML createSQLXML() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public boolean isValid(int timeout) throws SQLException { - if (timeout < 0) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - // TODO: - /* The driver shall submit a query on the connection or use some other mechanism that positively verifies - the connection is still valid when this method is called.*/ - return !isClosed(); - } - - @Override - public void setClientInfo(String name, String value) throws SQLClientInfoException { - if (isClosed) - throw new SQLClientInfoException(); - clientInfoProps.setProperty(name, value); - } - - @Override - public void setClientInfo(Properties properties) throws SQLClientInfoException { - if (isClosed) - throw new SQLClientInfoException(); - - for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { - String name = (String) enumer.nextElement(); - clientInfoProps.put(name, properties.getProperty(name)); - } - } - - @Override - public String getClientInfo(String name) throws SQLException { - if (isClosed) - throw new SQLClientInfoException(); - - return clientInfoProps.getProperty(name); - } - - @Override - public Properties getClientInfo() throws SQLException { - if (isClosed) - throw new SQLClientInfoException(); - - return clientInfoProps; - } - - @Override - public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void setSchema(String schema) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - synchronized (RestfulConnection.class) { - this.database = schema; - } - } - - @Override - public String getSchema() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - return this.database; - } - - @Override - public void abort(Executor executor) throws SQLException { - if (executor == null) { - throw new SQLException("Executor can not be null"); - } - - executor.execute(() -> { - try { - close(); - } catch (SQLException e) { - e.printStackTrace(); - } - }); - } - - @Override - public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public int getNetworkTimeout() throws SQLException { - if (isClosed()) - throw new SQLException(CONNECTION_IS_CLOSED); - return 0; - } - - @Override - public T unwrap(Class iface) throws SQLException { - try { - return iface.cast(this); - } catch (ClassCastException cce) { - throw new SQLException("Unable to unwrap to " + iface.toString()); - } - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return iface.isInstance(this); - } - public String getHost() { return host; } @@ -474,10 +73,6 @@ public class RestfulConnection implements Connection { return port; } - public Properties getProps() { - return props; - } - public String getDatabase() { return database; } @@ -485,4 +80,4 @@ public class RestfulConnection implements Connection { public String getUrl() { return url; } -} +} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java index 21d2c6402f820f6548b9fb954f83e14a02065d20..d108f46a796bbae350f942a17f90efa29cc3021e 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaData.java @@ -1,14 +1,15 @@ package com.taosdata.jdbc.rs; -import com.taosdata.jdbc.*; +import com.taosdata.jdbc.AbstractDatabaseMetaData; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; -import java.sql.*; -import java.util.ArrayList; -import java.util.List; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; public class RestfulDatabaseMetaData extends AbstractDatabaseMetaData { - private final String url; private final String userName; private final Connection connection; @@ -30,177 +31,57 @@ public class RestfulDatabaseMetaData extends AbstractDatabaseMetaData { } @Override - public String getSchemaTerm() throws SQLException { - return null; - } - - @Override - public String getProcedureTerm() throws SQLException { - return null; - } - - @Override - public String getCatalogTerm() throws SQLException { - return null; - } - - @Override - public boolean isCatalogAtStart() throws SQLException { - return false; + public String getDriverName() throws SQLException { + return RestfulDriver.class.getName(); } @Override public ResultSet getTables(String catalog, String schemaPattern, String tableNamePattern, String[] types) throws SQLException { - Statement stmt = null; - if (null != connection && !connection.isClosed()) { - stmt = connection.createStatement(); - if (catalog == null || catalog.length() < 1) { - catalog = connection.getCatalog(); - } - stmt.executeUpdate("use " + catalog); - ResultSet resultSet0 = stmt.executeQuery("show tables"); - GetTablesResultSet getTablesResultSet = new GetTablesResultSet(resultSet0, catalog, schemaPattern, tableNamePattern, types); - return getTablesResultSet; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + if (connection == null || connection.isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + return super.getTables(catalog, schemaPattern, tableNamePattern, types, connection); } @Override public ResultSet getCatalogs() throws SQLException { - if (connection != null && !connection.isClosed()) { - Statement stmt = connection.createStatement(); - ResultSet resultSet0 = stmt.executeQuery("show databases"); - CatalogResultSet resultSet = new CatalogResultSet(resultSet0); - return resultSet; - } else { - return new EmptyResultSet(); - } + if (connection == null || connection.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getCatalogs(connection); } @Override - public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { - Statement stmt = null; - if (null != connection && !connection.isClosed()) { - stmt = connection.createStatement(); - if (catalog == null || catalog.length() < 1) { - catalog = connection.getCatalog(); - } - stmt.execute("use " + catalog); - - DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); - // set up ColumnMetaDataList - List columnMetaDataList = new ArrayList<>(24); - columnMetaDataList.add(null); - columnMetaDataList.add(null); - // add TABLE_NAME - ColumnMetaData colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(3); - colMetaData.setColName("TABLE_NAME"); - colMetaData.setColSize(193); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add COLUMN_NAME - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(4); - colMetaData.setColName("COLUMN_NAME"); - colMetaData.setColSize(65); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add DATA_TYPE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(5); - colMetaData.setColName("DATA_TYPE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add TYPE_NAME - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(6); - colMetaData.setColName("TYPE_NAME"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_BINARY); - columnMetaDataList.add(colMetaData); - // add COLUMN_SIZE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(7); - colMetaData.setColName("COLUMN_SIZE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add BUFFER_LENGTH ,not used - columnMetaDataList.add(null); - // add DECIMAL_DIGITS - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(9); - colMetaData.setColName("DECIMAL_DIGITS"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add NUM_PREC_RADIX - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(10); - colMetaData.setColName("NUM_PREC_RADIX"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - // add NULLABLE - colMetaData = new ColumnMetaData(); - colMetaData.setColIndex(11); - colMetaData.setColName("NULLABLE"); - colMetaData.setColType(TSDBConstants.TSDB_DATA_TYPE_INT); - columnMetaDataList.add(colMetaData); - - resultSet.setColumnMetaDataList(columnMetaDataList); - - // set up rowDataList - ResultSet resultSet0 = stmt.executeQuery("describe " + tableNamePattern); - List rowDataList = new ArrayList<>(); - int index = 0; - while (resultSet0.next()) { - TSDBResultSetRowData rowData = new TSDBResultSetRowData(24); - // set TABLE_NAME - rowData.setString(2, tableNamePattern); - // set COLUMN_NAME - rowData.setString(3, resultSet0.getString(1)); - // set DATA_TYPE - String typeName = resultSet0.getString(2); - rowData.setInt(4, getDataType(typeName)); - // set TYPE_NAME - rowData.setString(5, typeName); - // set COLUMN_SIZE - int length = resultSet0.getInt(3); - rowData.setInt(6, getColumnSize(typeName, length)); - // set DECIMAL_DIGITS - rowData.setInt(8, getDecimalDigits(typeName)); - // set NUM_PREC_RADIX - rowData.setInt(9, 10); - // set NULLABLE - rowData.setInt(10, getNullable(index, typeName)); - rowDataList.add(rowData); - index++; - } - resultSet.setRowDataList(rowDataList); - - return resultSet; - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + public ResultSet getTableTypes() throws SQLException { + if (connection == null || connection.isClosed()) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); } + return super.getTableTypes(); } @Override - public long getMaxLogicalLobSize() throws SQLException { - return 0; + public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException { + if (connection == null || connection.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getColumns(catalog, schemaPattern, tableNamePattern, columnNamePattern, connection); } @Override - public boolean supportsRefCursors() throws SQLException { - return false; + public ResultSet getPrimaryKeys(String catalog, String schema, String table) throws SQLException { + if (connection == null || connection.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getPrimaryKeys(catalog, schema, table, connection); } - @Override - public T unwrap(Class iface) throws SQLException { - return null; + public Connection getConnection() throws SQLException { + return this.connection; } @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return false; + public ResultSet getSuperTables(String catalog, String schemaPattern, String tableNamePattern) throws SQLException { + if (connection == null || connection.isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_CONNECTION_CLOSED); + return super.getSuperTables(catalog, schemaPattern, tableNamePattern, connection); } + } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index cb6ff369f2b43f7cfae63fcc57701d692d3d803b..6efe13561d9509ee00d453e731bbc1ae4e3f0a75 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -2,32 +2,33 @@ package com.taosdata.jdbc.rs; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; -import com.taosdata.jdbc.AbstractTaosDriver; -import com.taosdata.jdbc.TSDBConstants; -import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.rs.util.HttpClientPoolUtil; +import com.taosdata.jdbc.*; +import com.taosdata.jdbc.utils.HttpClientPoolUtil; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.sql.*; import java.util.Properties; import java.util.logging.Logger; -public class RestfulDriver extends AbstractTaosDriver { +public class RestfulDriver extends AbstractDriver { private static final String URL_PREFIX = "jdbc:TAOS-RS://"; static { try { - DriverManager.registerDriver(new RestfulDriver()); + java.sql.DriverManager.registerDriver(new RestfulDriver()); } catch (SQLException e) { - throw new RuntimeException(TSDBConstants.WrapErrMsg("can not register Restful JDBC driver"), e); + throw TSDBError.createRuntimeException(TSDBErrorNumbers.ERROR_URL_NOT_SET, e); } } @Override public Connection connect(String url, Properties info) throws SQLException { // throw SQLException if url is null - if (url == null) - throw new SQLException(TSDBConstants.WrapErrMsg("url is not set!")); + if (url == null || url.trim().isEmpty() || url.trim().replaceAll("\\s", "").isEmpty()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_URL_NOT_SET); + // return null if url is not be accepted if (!acceptsURL(url)) return null; @@ -41,21 +42,38 @@ public class RestfulDriver extends AbstractTaosDriver { + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + props.getProperty(TSDBDriver.PROPERTY_KEY_USER) + "/" + props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD) + ""; + try { + String user = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_USER), "UTF-8"); + String password = URLEncoder.encode(props.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD), "UTF-8"); + loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" + user + "/" + password + ""; + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + String result = HttpClientPoolUtil.execute(loginUrl); JSONObject jsonResult = JSON.parseObject(result); String status = jsonResult.getString("status"); + String token = jsonResult.getString("desc"); + HttpClientPoolUtil.token = token; if (!status.equals("succ")) { throw new SQLException(jsonResult.getString("desc")); } - return new RestfulConnection(host, port, props, database, url); + RestfulConnection conn = new RestfulConnection(host, port, props, database, url); + if (database != null && !database.trim().replaceAll("\\s", "").isEmpty()) { + Statement stmt = conn.createStatement(); + stmt.execute("use " + database); + stmt.close(); + } + return conn; } @Override public boolean acceptsURL(String url) throws SQLException { if (url == null) - throw new SQLException(TSDBConstants.WrapErrMsg("url is null")); - return (url != null && url.length() > 0 && url.trim().length() > 0) && url.startsWith(URL_PREFIX); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_URL_NOT_SET); + return url.length() > 0 && url.trim().length() > 0 && url.startsWith(URL_PREFIX); } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java new file mode 100644 index 0000000000000000000000000000000000000000..7a130eb72b528de0ef61aa94e69ab376d9d214ba --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulParameterMetaData.java @@ -0,0 +1,10 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.AbstractParameterMetaData; + +public class RestfulParameterMetaData extends AbstractParameterMetaData { + + RestfulParameterMetaData(Object[] parameters) { + super(parameters); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java new file mode 100644 index 0000000000000000000000000000000000000000..f82955ca9dae5e3712da511a576aee64528061b5 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulPreparedStatement.java @@ -0,0 +1,472 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.net.URL; +import java.nio.charset.Charset; +import java.sql.*; +import java.util.Calendar; + +public class RestfulPreparedStatement extends RestfulStatement implements PreparedStatement { + + private ParameterMetaData parameterMetaData; + private final String rawSql; + private Object[] parameters; + private boolean isPrepared; + + public RestfulPreparedStatement(RestfulConnection conn, String database, String sql) { + super(conn, database); + this.rawSql = sql; + if (sql.contains("?")) { + int parameterCnt = 0; + for (int i = 0; i < sql.length(); i++) { + if ('?' == sql.charAt(i)) { + parameterCnt++; + } + } + parameters = new Object[parameterCnt]; + this.isPrepared = true; + } + + // build parameterMetaData + this.parameterMetaData = new RestfulParameterMetaData(parameters); + } + + @Override + public ResultSet executeQuery() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!isPrepared) + return executeQuery(this.rawSql); + + final String sql = getNativeSql(this.rawSql); + return executeQuery(sql); + } + + @Override + public int executeUpdate() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!isPrepared) + return executeUpdate(this.rawSql); + + final String sql = getNativeSql(this.rawSql); + return executeUpdate(sql); + } + + private String getNativeSql(String rawSql) throws SQLException { + String sql = rawSql; + for (int i = 0; i < parameters.length; ++i) { + Object para = parameters[i]; + if (para != null) { + String paraStr; + if (para instanceof byte[]) { + paraStr = new String((byte[]) para, Charset.forName("UTF-8")); + } else { + paraStr = para.toString(); + } + // if para is timestamp or String or byte[] need to translate ' character + if (para instanceof Timestamp || para instanceof String || para instanceof byte[]) { + paraStr = paraStr.replaceAll("'", "\\\\\\\\'"); + paraStr = "'" + paraStr + "'"; + } + sql = sql.replaceFirst("[?]", paraStr); + } else { + sql = sql.replaceFirst("[?]", "NULL"); + } + } + clearParameters(); + return sql; + } + + @Override + public void setNull(int parameterIndex, int sqlType) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex, "NULL"); + } + + @Override + public void setBoolean(int parameterIndex, boolean x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setByte(int parameterIndex, byte x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex, x); + } + + @Override + public void setShort(int parameterIndex, short x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setInt(int parameterIndex, int x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex, x); + } + + @Override + public void setLong(int parameterIndex, long x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setFloat(int parameterIndex, float x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setDouble(int parameterIndex, double x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setBigDecimal(int parameterIndex, BigDecimal x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setString(int parameterIndex, String x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setBytes(int parameterIndex, byte[] x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + setObject(parameterIndex, x); + } + + @Override + public void setDate(int parameterIndex, Date x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTime(int parameterIndex, Time x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex, x); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setUnicodeStream(int parameterIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void clearParameters() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + this.parameters = new Object[parameters.length]; + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + setObject(parameterIndex,x); + } + + @Override + public void setObject(int parameterIndex, Object x) throws SQLException { + if (parameterIndex < 1 && parameterIndex >= parameters.length) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_PARAMETER_INDEX_OUT_RANGE); + + parameters[parameterIndex - 1] = x; + } + + @Override + public boolean execute() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!isPrepared) + return execute(this.rawSql); + final String sql = getNativeSql(rawSql); + return execute(sql); + } + + @Override + public void addBatch() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + + final String sql = getNativeSql(this.rawSql); + addBatch(sql); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, int length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setRef(int parameterIndex, Ref x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBlob(int parameterIndex, Blob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setClob(int parameterIndex, Clob x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setArray(int parameterIndex, Array x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public ResultSetMetaData getMetaData() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setDate(int parameterIndex, Date x, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTime(int parameterIndex, Time x, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setTimestamp(int parameterIndex, Timestamp x, Calendar cal) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNull(int parameterIndex, int sqlType, String typeName) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setURL(int parameterIndex, URL x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public ParameterMetaData getParameterMetaData() throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); +// throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + + return this.parameterMetaData; + } + + @Override + public void setRowId(int parameterIndex, RowId x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNString(int parameterIndex, String value) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNClob(int parameterIndex, NClob value) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setClob(int parameterIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNClob(int parameterIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setSQLXML(int parameterIndex, SQLXML xmlObject) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setObject(int parameterIndex, Object x, int targetSqlType, int scaleOrLength) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader, long length) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setAsciiStream(int parameterIndex, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBinaryStream(int parameterIndex, InputStream x) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setCharacterStream(int parameterIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNCharacterStream(int parameterIndex, Reader value) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setClob(int parameterIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setBlob(int parameterIndex, InputStream inputStream) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } + + @Override + public void setNClob(int parameterIndex, Reader reader) throws SQLException { + if (isClosed()) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 1aa3d5b3cefe2524f0246b500af6687a79d6b20c..5c2d4c45b079974672843b232e38fcd9d010b0c4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -2,120 +2,118 @@ package com.taosdata.jdbc.rs; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.Shorts; +import com.taosdata.jdbc.AbstractResultSet; import com.taosdata.jdbc.TSDBConstants; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; -import java.io.InputStream; -import java.io.Reader; import java.math.BigDecimal; -import java.net.URL; import java.sql.*; import java.util.ArrayList; import java.util.Calendar; -import java.util.List; -import java.util.Map; -public class RestfulResultSet implements ResultSet { - private static final String RESULT_SET_IS_CLOSED = "resultSet is closed."; +public class RestfulResultSet extends AbstractResultSet implements ResultSet { private volatile boolean isClosed; private int pos = -1; private final String database; private final Statement statement; // data - private ArrayList> resultSet; + private final ArrayList> resultSet; // meta private ArrayList columnNames; private ArrayList columns; private RestfulResultSetMetaData metaData; /** - * 由一个result的Json构造结果集,对应执行show databases,show tables等这些语句,返回结果集,但无法获取结果集对应的meta,统一当成String处理 + * 由一个result的Json构造结果集,对应执行show databases, show tables等这些语句,返回结果集,但无法获取结果集对应的meta,统一当成String处理 + * + * @param resultJson: 包含data信息的结果集,有sql返回的结果集 ***/ - public RestfulResultSet(String database, Statement statement, JSONObject resultJson) { + public RestfulResultSet(String database, Statement statement, JSONObject resultJson) throws SQLException { this.database = database; this.statement = statement; - // row data - JSONArray data = resultJson.getJSONArray("data"); - resultSet = new ArrayList<>(); - int columnIndex = 0; - for (; columnIndex < data.size(); columnIndex++) { - ArrayList oneRow = new ArrayList<>(); - JSONArray one = data.getJSONArray(columnIndex); - for (int j = 0; j < one.size(); j++) { - oneRow.add(one.getString(j)); - } - resultSet.add(oneRow); - } - // column only names + // column metadata + JSONArray columnMeta = resultJson.getJSONArray("column_meta"); columnNames = new ArrayList<>(); columns = new ArrayList<>(); - JSONArray head = resultJson.getJSONArray("head"); - for (int i = 0; i < head.size(); i++) { - String name = head.getString(i); - columnNames.add(name); - columns.add(new Field(name, "", 0, "")); + for (int colIndex = 0; colIndex < columnMeta.size(); colIndex++) { + JSONArray col = columnMeta.getJSONArray(colIndex); + String col_name = col.getString(0); + int taos_type = col.getInteger(1); + int col_type = TSDBConstants.taosType2JdbcType(taos_type); + int col_length = col.getInteger(2); + columnNames.add(col_name); + columns.add(new Field(col_name, col_type, col_length, "", taos_type)); } - this.metaData = new RestfulResultSetMetaData(this.database, columns); - } + this.metaData = new RestfulResultSetMetaData(this.database, columns, this); - /** - * 由多个resultSet的JSON构造结果集 - * - * @param resultJson: 包含data信息的结果集,有sql返回的结果集 - * @param fieldJson: 包含多个(最多2个)meta信息的结果集,有describe xxx - **/ - public RestfulResultSet(String database, Statement statement, JSONObject resultJson, List fieldJson) { - this(database, statement, resultJson); - ArrayList newColumns = new ArrayList<>(); - - for (Field column : columns) { - Field field = findField(column.name, fieldJson); - if (field != null) { - newColumns.add(field); - } else { - newColumns.add(column); + // row data + JSONArray data = resultJson.getJSONArray("data"); + resultSet = new ArrayList<>(); + for (int rowIndex = 0; rowIndex < data.size(); rowIndex++) { + ArrayList row = new ArrayList(); + JSONArray jsonRow = data.getJSONArray(rowIndex); + for (int colIndex = 0; colIndex < jsonRow.size(); colIndex++) { + row.add(parseColumnData(jsonRow, colIndex, columns.get(colIndex).taos_type)); } + resultSet.add(row); } - this.columns = newColumns; - this.metaData = new RestfulResultSetMetaData(this.database, this.columns); } - public Field findField(String columnName, List fieldJsonList) { - for (JSONObject fieldJSON : fieldJsonList) { - JSONArray fieldDataJson = fieldJSON.getJSONArray("data"); - for (int i = 0; i < fieldDataJson.size(); i++) { - JSONArray field = fieldDataJson.getJSONArray(i); - if (columnName.equalsIgnoreCase(field.getString(0))) { - return new Field(field.getString(0), field.getString(1), field.getInteger(2), field.getString(3)); - } - } + private Object parseColumnData(JSONArray row, int colIndex, int taosType) { + switch (taosType) { + case TSDBConstants.TSDB_DATA_TYPE_BOOL: + return row.getBoolean(colIndex); + case TSDBConstants.TSDB_DATA_TYPE_TINYINT: + return row.getByte(colIndex); + case TSDBConstants.TSDB_DATA_TYPE_SMALLINT: + return row.getShort(colIndex); + case TSDBConstants.TSDB_DATA_TYPE_INT: + return row.getInteger(colIndex); + case TSDBConstants.TSDB_DATA_TYPE_BIGINT: + return row.getLong(colIndex); + case TSDBConstants.TSDB_DATA_TYPE_FLOAT: + return row.getFloat(colIndex); + case TSDBConstants.TSDB_DATA_TYPE_DOUBLE: + return row.getDouble(colIndex); + case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: + return new Timestamp(row.getDate(colIndex).getTime()); + case TSDBConstants.TSDB_DATA_TYPE_BINARY: + return row.getString(colIndex) == null ? null : row.getString(colIndex).getBytes(); + case TSDBConstants.TSDB_DATA_TYPE_NCHAR: + return row.getString(colIndex) == null ? null : row.getString(colIndex); + default: + return row.get(colIndex); } - - return null; } public class Field { String name; - String type; + int type; int length; String note; + int taos_type; - public Field(String name, String type, int length, String note) { + public Field(String name, int type, int length, String note, int taos_type) { this.name = name; this.type = type; this.length = length; this.note = note; + this.taos_type = taos_type; } } @Override public boolean next() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - if (pos < resultSet.size() - 1) { - pos++; + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + pos++; + if (pos <= resultSet.size() - 1) { return true; } return false; @@ -131,1104 +129,399 @@ public class RestfulResultSet implements ResultSet { @Override public boolean wasNull() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); return resultSet.isEmpty(); } @Override public String getString(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); - if (columnIndex > resultSet.get(pos).size()) { - throw new SQLException(TSDBConstants.WrapErrMsg("Column Index out of range, " + columnIndex + " > " + resultSet.get(pos).size())); - } - return resultSet.get(pos).get(columnIndex - 1).toString(); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof byte[]) + return new String((byte[]) value); + return value.toString(); } @Override public boolean getBoolean(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); - String result = getString(columnIndex); - if (!(result.equals("true") || result.equals("false"))) { - throw new SQLException("not boolean value"); - } - return result.equals("true"); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return false; + if (value instanceof Boolean) + return (boolean) value; + return Boolean.valueOf(value.toString()); } @Override public byte getByte(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); + + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return 0; + long valueAsLong = Long.parseLong(value.toString()); + if (valueAsLong == Byte.MIN_VALUE) + return 0; + if (valueAsLong < Byte.MIN_VALUE || valueAsLong > Byte.MAX_VALUE) + throwRangeException(value.toString(), columnIndex, Types.TINYINT); + + return (byte) valueAsLong; + } - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + private void throwRangeException(String valueAsString, int columnIndex, int jdbcType) throws SQLException { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_NUMERIC_VALUE_OUT_OF_RANGE, + "'" + valueAsString + "' in column '" + columnIndex + "' is outside valid range for the jdbcType " + TSDBConstants.jdbcType2TaosTypeName(jdbcType)); } @Override public short getShort(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); - return Short.parseShort(getString(columnIndex)); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return 0; + long valueAsLong = Long.parseLong(value.toString()); + if (valueAsLong == Short.MIN_VALUE) + return 0; + if (valueAsLong < Short.MIN_VALUE || valueAsLong > Short.MAX_VALUE) + throwRangeException(value.toString(), columnIndex, Types.SMALLINT); + return (short) valueAsLong; } @Override public int getInt(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - return Integer.parseInt(getString(columnIndex)); + checkAvailability(columnIndex, resultSet.get(pos).size()); + + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return 0; + long valueAsLong = Long.parseLong(value.toString()); + if (valueAsLong == Integer.MIN_VALUE) + return 0; + if (valueAsLong < Integer.MIN_VALUE || valueAsLong > Integer.MAX_VALUE) + throwRangeException(value.toString(), columnIndex, Types.INTEGER); + return (int) valueAsLong; } @Override public long getLong(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return Long.parseLong(getString(columnIndex)); + checkAvailability(columnIndex, resultSet.get(pos).size()); + + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return 0; + + long valueAsLong = 0; + try { + valueAsLong = Long.parseLong(value.toString()); + if (valueAsLong == Long.MIN_VALUE) + return 0; + } catch (NumberFormatException e) { + throwRangeException(value.toString(), columnIndex, Types.BIGINT); + } + return valueAsLong; } @Override public float getFloat(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); - return Float.parseFloat(getString(columnIndex)); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return 0; + if (value instanceof Float || value instanceof Double) + return (float) value; + return Float.parseFloat(value.toString()); } @Override public double getDouble(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return Double.parseDouble(getString(columnIndex)); - } - - /*******************************************************************************************************************/ + checkAvailability(columnIndex, resultSet.get(pos).size()); - @Override - public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return 0; + if (value instanceof Double || value instanceof Float) + return (double) value; + return Double.parseDouble(value.toString()); } @Override public byte[] getBytes(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof byte[]) + return (byte[]) value; + if (value instanceof String) + return ((String) value).getBytes(); + if (value instanceof Long) + return Longs.toByteArray((long) value); + if (value instanceof Integer) + return Ints.toByteArray((int) value); + if (value instanceof Short) + return Shorts.toByteArray((short) value); + if (value instanceof Byte) + return new byte[]{(byte) value}; + + return value.toString().getBytes(); } @Override public Date getDate(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof Timestamp) + return new Date(((Timestamp) value).getTime()); + return Date.valueOf(value.toString()); } @Override public Time getTime(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof Timestamp) + return new Time(((Timestamp) value).getTime()); + return Time.valueOf(value.toString()); } @Override public Timestamp getTimestamp(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - String strDate = getString(columnIndex); - strDate = strDate.substring(1, strDate.length() - 1); - return Timestamp.valueOf(strDate); - } - - @Override - public InputStream getAsciiStream(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public InputStream getUnicodeStream(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public InputStream getBinaryStream(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public String getString(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return getString(findColumn(columnLabel) + 1); - } - - @Override - public boolean getBoolean(String columnLabel) throws SQLException { - return getBoolean(findColumn(columnLabel)); - } - - @Override - public byte getByte(String columnLabel) throws SQLException { - return getByte(findColumn(columnLabel)); - } - - @Override - public short getShort(String columnLabel) throws SQLException { - return getShort(findColumn(columnLabel)); - } - - @Override - public int getInt(String columnLabel) throws SQLException { - return getInt(findColumn(columnLabel)); - } - - @Override - public long getLong(String columnLabel) throws SQLException { - return getLong(findColumn(columnLabel)); - } - - @Override - public float getFloat(String columnLabel) throws SQLException { - return getFloat(findColumn(columnLabel)); - } - - @Override - public double getDouble(String columnLabel) throws SQLException { - return getDouble(findColumn(columnLabel)); - } - - @Override - public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException { - return getBigDecimal(findColumn(columnLabel)); - } - - @Override - public byte[] getBytes(String columnLabel) throws SQLException { - return getBytes(findColumn(columnLabel)); - } - - @Override - public Date getDate(String columnLabel) throws SQLException { - return getDate(findColumn(columnLabel)); - } - - @Override - public Time getTime(String columnLabel) throws SQLException { - return getTime(findColumn(columnLabel)); - } - - @Override - public Timestamp getTimestamp(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - return Timestamp.valueOf(getString(findColumn(columnLabel))); - } - - @Override - public InputStream getAsciiStream(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public InputStream getUnicodeStream(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public InputStream getBinaryStream(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public SQLWarning getWarnings() throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - return null; - } - - @Override - public void clearWarnings() throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - return; - } - - @Override - public String getCursorName() throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + checkAvailability(columnIndex, resultSet.get(pos).size()); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; + if (value instanceof Timestamp) + return (Timestamp) value; + return Timestamp.valueOf(value.toString()); } @Override public ResultSetMetaData getMetaData() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); return this.metaData; } @Override public Object getObject(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + checkAvailability(columnIndex, resultSet.get(pos).size()); - @Override - public Object getObject(String columnLabel) throws SQLException { - return getObject(findColumn(columnLabel)); + return resultSet.get(pos).get(columnIndex - 1); } @Override public int findColumn(String columnLabel) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return columnNames.indexOf(columnLabel); - } - - @Override - public Reader getCharacterStream(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Reader getCharacterStream(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + int columnIndex = columnNames.indexOf(columnLabel); + if (columnIndex == -1) + throw new SQLException("cannot find Column in resultSet"); + return columnIndex + 1; } @Override public BigDecimal getBigDecimal(int columnIndex) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + checkAvailability(columnIndex, resultSet.get(pos).size()); - @Override - public BigDecimal getBigDecimal(String columnLabel) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + Object value = resultSet.get(pos).get(columnIndex - 1); + if (value == null) + return null; - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (value instanceof Long || value instanceof Integer || value instanceof Short || value instanceof Byte) + return new BigDecimal(Long.valueOf(value.toString())); + if (value instanceof Double || value instanceof Float) + return new BigDecimal(Double.valueOf(value.toString())); + if (value instanceof Timestamp) + return new BigDecimal(((Timestamp) value).getTime()); + return new BigDecimal(value.toString()); } @Override public boolean isBeforeFirst() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + return this.pos == -1 && this.resultSet.size() != 0; } @Override public boolean isAfterLast() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + + return this.pos >= resultSet.size() && this.resultSet.size() != 0; } @Override public boolean isFirst() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + return this.pos == 0; } @Override public boolean isLast() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + if (this.resultSet.size() == 0) + return false; + return this.pos == (this.resultSet.size() - 1); } @Override public void beforeFirst() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + synchronized (this) { + if (this.resultSet.size() > 0) { + this.pos = -1; + } + } } @Override public void afterLast() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + synchronized (this) { + if (this.resultSet.size() > 0) { + this.pos = this.resultSet.size(); + } + } } @Override public boolean first() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + if (this.resultSet.size() == 0) + return false; + + synchronized (this) { + this.pos = 0; + } + return true; } @Override public boolean last() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + if (this.resultSet.size() == 0) + return false; + synchronized (this) { + this.pos = this.resultSet.size() - 1; + } + return true; } @Override public int getRow() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + int row; + synchronized (this) { + if (this.pos < 0 || this.pos >= this.resultSet.size()) + return 0; + row = this.pos + 1; + } + return row; } @Override public boolean absolute(int row) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); + +// if (this.resultSet.size() == 0) +// return false; +// +// if (row == 0) { +// beforeFirst(); +// return false; +// } else if (row == 1) { +// return first(); +// } else if (row == -1) { +// return last(); +// } else if (row > this.resultSet.size()) { +// afterLast(); +// return false; +// } else { +// if (row < 0) { +// // adjust to reflect after end of result set +// int newRowPosition = this.resultSet.size() + row + 1; +// if (newRowPosition <= 0) { +// beforeFirst(); +// return false; +// } else { +// return absolute(newRowPosition); +// } +// } else { +// row--; // adjust for index difference +// this.pos = row; +// return true; +// } +// } + + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean relative(int rows) throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override public boolean previous() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void setFetchDirection(int direction) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - if (direction != ResultSet.FETCH_REVERSE || direction != ResultSet.FETCH_REVERSE || direction != ResultSet.FETCH_UNKNOWN) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public int getFetchDirection() throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - return ResultSet.FETCH_FORWARD; + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_UNSUPPORTED_METHOD); } @Override - public void setFetchSize(int rows) throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - if (rows < 0) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + public String getNString(int columnIndex) throws SQLException { + return getString(columnIndex); } @Override - public int getFetchSize() throws SQLException { + public Statement getStatement() throws SQLException { if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return this.resultSet.size(); - } - - @Override - public int getType() throws SQLException { - return ResultSet.TYPE_FORWARD_ONLY; - } - - @Override - public int getConcurrency() throws SQLException { - return ResultSet.CONCUR_READ_ONLY; - } - - @Override - public boolean rowUpdated() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public boolean rowInserted() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public boolean rowDeleted() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNull(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBoolean(int columnIndex, boolean x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateByte(int columnIndex, byte x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateShort(int columnIndex, short x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateInt(int columnIndex, int x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateLong(int columnIndex, long x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateFloat(int columnIndex, float x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateDouble(int columnIndex, double x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateString(int columnIndex, String x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBytes(int columnIndex, byte[] x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateDate(int columnIndex, Date x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateTime(int columnIndex, Time x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_RESULTSET_CLOSED); - @Override - public void updateObject(int columnIndex, Object x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + return this.statement; } @Override - public void updateNull(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { + //TODO:did not use the specified timezone in cal + return getTimestamp(columnIndex); } @Override - public void updateBoolean(String columnLabel, boolean x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + public boolean isClosed() throws SQLException { + return isClosed; } - @Override - public void updateByte(String columnLabel, byte x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - @Override - public void updateShort(String columnLabel, short x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateInt(String columnLabel, int x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateLong(String columnLabel, long x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateFloat(String columnLabel, float x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateDouble(String columnLabel, double x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateString(String columnLabel, String x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBytes(String columnLabel, byte[] x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateDate(String columnLabel, Date x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateTime(String columnLabel, Time x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateObject(String columnLabel, Object x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void insertRow() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateRow() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void deleteRow() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void refreshRow() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void cancelRowUpdates() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void moveToInsertRow() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void moveToCurrentRow() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Statement getStatement() throws SQLException { - if (isClosed()) - throw new SQLException(TSDBConstants.WrapErrMsg(RESULT_SET_IS_CLOSED)); - - return this.statement; - } - - @Override - public Object getObject(int columnIndex, Map> map) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Ref getRef(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Blob getBlob(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Clob getClob(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Array getArray(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - /******************************************************************************************************************/ - @Override - public Object getObject(String columnLabel, Map> map) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Ref getRef(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Blob getBlob(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Clob getClob(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Array getArray(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Date getDate(int columnIndex, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Date getDate(String columnLabel, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Time getTime(int columnIndex, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Time getTime(String columnLabel, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public URL getURL(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public URL getURL(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateRef(int columnIndex, Ref x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateRef(String columnLabel, Ref x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBlob(int columnIndex, Blob x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBlob(String columnLabel, Blob x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateClob(int columnIndex, Clob x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateClob(String columnLabel, Clob x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateArray(int columnIndex, Array x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateArray(String columnLabel, Array x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public RowId getRowId(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public RowId getRowId(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateRowId(int columnIndex, RowId x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateRowId(String columnLabel, RowId x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public int getHoldability() throws SQLException { - return ResultSet.HOLD_CURSORS_OVER_COMMIT; - } - - @Override - public boolean isClosed() throws SQLException { - return false; - //TODO: SQLFeature Not Supported -// throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNString(int columnIndex, String nString) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNString(String columnLabel, String nString) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNClob(int columnIndex, NClob nClob) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNClob(String columnLabel, NClob nClob) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public NClob getNClob(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public NClob getNClob(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public SQLXML getSQLXML(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public SQLXML getSQLXML(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public String getNString(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public String getNString(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Reader getNCharacterStream(int columnIndex) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public Reader getNCharacterStream(String columnLabel) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateClob(int columnIndex, Reader reader, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateClob(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateCharacterStream(int columnIndex, Reader x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateClob(int columnIndex, Reader reader) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateClob(String columnLabel, Reader reader) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNClob(int columnIndex, Reader reader) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public void updateNClob(String columnLabel, Reader reader) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public T getObject(int columnIndex, Class type) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public T getObject(String columnLabel, Class type) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public T unwrap(Class iface) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java index 1af3088b172f751ef40be5dd92b205949bc009d6..7ead8bd1bbaeeb8fe1f24c951656888f6bba6b6f 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSetMetaData.java @@ -1,17 +1,28 @@ package com.taosdata.jdbc.rs; +import com.taosdata.jdbc.TSDBConstants; +import com.taosdata.jdbc.WrapperImpl; + import java.sql.ResultSetMetaData; import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; import java.util.ArrayList; -public class RestfulResultSetMetaData implements ResultSetMetaData { +public class RestfulResultSetMetaData extends WrapperImpl implements ResultSetMetaData { private final String database; private ArrayList fields; + private final RestfulResultSet resultSet; - public RestfulResultSetMetaData(String database, ArrayList fields) { + public RestfulResultSetMetaData(String database, ArrayList fields, RestfulResultSet resultSet) { this.database = database; this.fields = fields; + this.resultSet = resultSet; + } + + public ArrayList getFields() { + return fields; } @Override @@ -26,13 +37,12 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public boolean isCaseSensitive(int column) throws SQLException { - //TODO return false; } @Override public boolean isSearchable(int column) throws SQLException { - return false; + return true; } @Override @@ -42,17 +52,30 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public int isNullable(int column) throws SQLException { + if (column == 1) + return ResultSetMetaData.columnNoNulls; return ResultSetMetaData.columnNullable; } @Override public boolean isSigned(int column) throws SQLException { - return false; + int type = this.fields.get(column - 1).type; + switch (type) { + case Types.TINYINT: + case Types.SMALLINT: + case Types.INTEGER: + case Types.BIGINT: + case Types.FLOAT: + case Types.DOUBLE: + return true; + default: + return false; + } } @Override public int getColumnDisplaySize(int column) throws SQLException { - return 0; + return this.fields.get(column - 1).length; } @Override @@ -62,27 +85,46 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public String getColumnName(int column) throws SQLException { - return null; + return fields.get(column - 1).name; } @Override public String getSchemaName(int column) throws SQLException { - return this.database; + return ""; } @Override public int getPrecision(int column) throws SQLException { - return 0; + int type = this.fields.get(column - 1).type; + switch (type) { + case Types.FLOAT: + return 5; + case Types.DOUBLE: + return 9; + case Types.BINARY: + case Types.NCHAR: + return this.fields.get(column - 1).length; + default: + return 0; + } } @Override public int getScale(int column) throws SQLException { - return 0; + int type = this.fields.get(column - 1).type; + switch (type) { + case Types.FLOAT: + return 5; + case Types.DOUBLE: + return 9; + default: + return 0; + } } @Override public String getTableName(int column) throws SQLException { - return null; + return ""; } @Override @@ -92,17 +134,18 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public int getColumnType(int column) throws SQLException { - return 0; + return this.fields.get(column - 1).type; } @Override public String getColumnTypeName(int column) throws SQLException { - return null; + int taos_type = fields.get(column - 1).taos_type; + return TSDBConstants.taosType2JdbcTypeName(taos_type); } @Override public boolean isReadOnly(int column) throws SQLException { - return false; + return true; } @Override @@ -117,16 +160,29 @@ public class RestfulResultSetMetaData implements ResultSetMetaData { @Override public String getColumnClassName(int column) throws SQLException { - return null; - } - - @Override - public T unwrap(Class iface) throws SQLException { - return null; + int type = this.fields.get(column - 1).type; + String columnClassName = ""; + switch (type) { + case Types.BOOLEAN: + return Boolean.class.getName(); + case Types.TINYINT: + case Types.SMALLINT: + return Short.class.getName(); + case Types.INTEGER: + return Integer.class.getName(); + case Types.BIGINT: + return Long.class.getName(); + case Types.FLOAT: + return Float.class.getName(); + case Types.DOUBLE: + return Double.class.getName(); + case Types.TIMESTAMP: + return Timestamp.class.getName(); + case Types.BINARY: + case Types.NCHAR: + return String.class.getName(); + } + return columnClassName; } - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return false; - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 8b2276fbb055cd18c86173f387fb7c93e296fe43..9071c046726a0ef7df3f712ac3ddbb10d3afafba 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -1,123 +1,64 @@ package com.taosdata.jdbc.rs; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; -import com.taosdata.jdbc.TSDBConstants; -import com.taosdata.jdbc.rs.util.HttpClientPoolUtil; +import com.taosdata.jdbc.AbstractStatement; +import com.taosdata.jdbc.TSDBError; +import com.taosdata.jdbc.TSDBErrorNumbers; +import com.taosdata.jdbc.utils.HttpClientPoolUtil; import com.taosdata.jdbc.utils.SqlSyntaxValidator; -import java.sql.*; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; -public class RestfulStatement implements Statement { +public class RestfulStatement extends AbstractStatement { - private static final String STATEMENT_CLOSED = "Statement already closed."; private boolean closed; private String database; private final RestfulConnection conn; private volatile RestfulResultSet resultSet; private volatile int affectedRows; - private volatile boolean closeOnCompletion; public RestfulStatement(RestfulConnection conn, String database) { this.conn = conn; this.database = database; } - private String[] parseTableIdentifier(String sql) { - sql = sql.trim().toLowerCase(); - String[] ret = null; - if (sql.contains("where")) - sql = sql.substring(0, sql.indexOf("where")); - if (sql.contains("interval")) - sql = sql.substring(0, sql.indexOf("interval")); - if (sql.contains("fill")) - sql = sql.substring(0, sql.indexOf("fill")); - if (sql.contains("sliding")) - sql = sql.substring(0, sql.indexOf("sliding")); - if (sql.contains("group by")) - sql = sql.substring(0, sql.indexOf("group by")); - if (sql.contains("order by")) - sql = sql.substring(0, sql.indexOf("order by")); - if (sql.contains("slimit")) - sql = sql.substring(0, sql.indexOf("slimit")); - if (sql.contains("limit")) - sql = sql.substring(0, sql.indexOf("limit")); - // parse - if (sql.contains("from")) { - sql = sql.substring(sql.indexOf("from") + 4).trim(); - return Arrays.asList(sql.split(",")).stream() - .map(tableIdentifier -> { - tableIdentifier = tableIdentifier.trim(); - if (tableIdentifier.contains(" ")) - tableIdentifier = tableIdentifier.substring(0, tableIdentifier.indexOf(" ")); - return tableIdentifier; - }).collect(Collectors.joining(",")).split(","); - } - return ret; - } - @Override public ResultSet executeQuery(String sql) throws SQLException { if (isClosed()) - throw new SQLException("statement already closed"); - if (!SqlSyntaxValidator.isSelectSql(sql)) - throw new SQLException("not a select sql for executeQuery: " + sql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql); final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; - // row data - String result = HttpClientPoolUtil.execute(url, sql); - JSONObject resultJson = JSON.parseObject(result); - if (resultJson.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + resultJson.getString("desc") + "\n" + "error code: " + resultJson.getString("code"))); - } - - // parse table name from sql - String[] tableIdentifiers = parseTableIdentifier(sql); - if (tableIdentifiers != null) { - List fieldJsonList = new ArrayList<>(); - for (String tableIdentifier : tableIdentifiers) { - // field meta - String fields = HttpClientPoolUtil.execute(url, "DESCRIBE " + tableIdentifier); - JSONObject fieldJson = JSON.parseObject(fields); - if (fieldJson.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + fieldJson.getString("desc") + "\n" + "error code: " + fieldJson.getString("code"))); - } - fieldJsonList.add(fieldJson); - } - this.resultSet = new RestfulResultSet(database, this, resultJson, fieldJsonList); - } else { - this.resultSet = new RestfulResultSet(database, this, resultJson); + if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { + return executeOneQuery(url, sql); } - this.affectedRows = 0; - return resultSet; +// if (this.database != null && !this.database.trim().replaceAll("\\s","").isEmpty()) +// HttpClientPoolUtil.execute(url, "use " + this.database); + return executeOneQuery(url, sql); } @Override public int executeUpdate(String sql) throws SQLException { if (isClosed()) - throw new SQLException("statement already closed"); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) - throw new SQLException("not a valid sql for executeUpdate: " + sql); - - if (this.database == null) - throw new SQLException("Database not specified or available"); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql); - final String url = "http://" + conn.getHost().trim() + ":" + conn.getPort() + "/rest/sql"; -// HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); - String result = HttpClientPoolUtil.execute(url, sql); - JSONObject jsonObject = JSON.parseObject(result); - if (jsonObject.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + jsonObject.getString("desc") + "\n" + "error code: " + jsonObject.getString("code"))); + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { + return executeOneUpdate(url, sql); } - this.resultSet = null; - this.affectedRows = Integer.parseInt(jsonObject.getString("rows")); - return this.affectedRows; + +// if (this.database != null && !this.database.trim().replaceAll("\\s", "").isEmpty()) +// HttpClientPoolUtil.execute(url, "use " + this.database); + return executeOneUpdate(url, sql); } @Override @@ -129,313 +70,105 @@ public class RestfulStatement implements Statement { } @Override - public int getMaxFieldSize() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return TSDBConstants.maxFieldSize; - } - - @Override - public void setMaxFieldSize(int max) throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - if (max < 0) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - // nothing to do - } - - @Override - public int getMaxRows() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return 0; - } - - @Override - public void setMaxRows(int max) throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - if (max < 0) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - // nothing to do - } - - @Override - public void setEscapeProcessing(boolean enable) throws SQLException { - if (isClosed()) - throw new SQLException(RestfulStatement.STATEMENT_CLOSED); - } - - @Override - public int getQueryTimeout() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return 0; - } - - @Override - public void setQueryTimeout(int seconds) throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - if (seconds < 0) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - @Override - public void cancel() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public SQLWarning getWarnings() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return null; - } - - @Override - public void clearWarnings() throws SQLException { - // nothing to do - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - } - - @Override - public void setCursorName(String name) throws SQLException { + public boolean execute(String sql) throws SQLException { if (isClosed()) - throw new SQLException(RestfulStatement.STATEMENT_CLOSED); - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + if (!SqlSyntaxValidator.isValidForExecute(sql)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE, "not a valid sql for execute: " + sql); - @Override - public boolean execute(String sql) throws SQLException { - if (isClosed()) { - throw new SQLException("Invalid method call on a closed statement."); - } //如果执行了use操作应该将当前Statement的catalog设置为新的database + boolean result = true; + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; if (SqlSyntaxValidator.isUseSql(sql)) { + HttpClientPoolUtil.execute(url, sql); this.database = sql.trim().replace("use", "").trim(); this.conn.setCatalog(this.database); - } - if (this.database == null) - throw new SQLException("Database not specified or available"); - - if (SqlSyntaxValidator.isSelectSql(sql)) { - executeQuery(sql); - } else if (SqlSyntaxValidator.isShowSql(sql) || SqlSyntaxValidator.isDescribeSql(sql)) { - final String url = "http://" + conn.getHost().trim() + ":" + conn.getPort() + "/rest/sql"; - if (!SqlSyntaxValidator.isShowDatabaseSql(sql)) { - HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); - } - String result = HttpClientPoolUtil.execute(url, sql); - JSONObject resultJson = JSON.parseObject(result); - if (resultJson.getString("status").equals("error")) { - throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + resultJson.getString("desc") + "\n" + "error code: " + resultJson.getString("code"))); - } - this.resultSet = new RestfulResultSet(database, this, resultJson); + result = false; + } else if (SqlSyntaxValidator.isDatabaseUnspecifiedQuery(sql)) { + executeOneQuery(url, sql); + } else if (SqlSyntaxValidator.isDatabaseUnspecifiedUpdate(sql)) { + executeOneUpdate(url, sql); + result = false; } else { - executeUpdate(sql); + if (SqlSyntaxValidator.isValidForExecuteQuery(sql)) { + executeQuery(sql); + } else { + executeUpdate(sql); + result = false; + } } - return true; + return result; } - @Override - public ResultSet getResultSet() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return resultSet; - } + private ResultSet executeOneQuery(String url, String sql) throws SQLException { + if (!SqlSyntaxValidator.isValidForExecuteQuery(sql)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_QUERY, "not a valid sql for executeQuery: " + sql); - @Override - public int getUpdateCount() throws SQLException { - if (isClosed()) { - throw new SQLException("Invalid method call on a closed statement."); + // row data + String result = HttpClientPoolUtil.execute(url, sql); + JSONObject resultJson = JSON.parseObject(result); + if (resultJson.getString("status").equals("error")) { + throw TSDBError.createSQLException(resultJson.getInteger("code"), resultJson.getString("desc")); } - return this.affectedRows; - } - - @Override - public boolean getMoreResults() throws SQLException { - return getMoreResults(CLOSE_CURRENT_RESULT); - } - - @Override - public void setFetchDirection(int direction) throws SQLException { - if (direction != ResultSet.FETCH_FORWARD && direction != ResultSet.FETCH_REVERSE && direction != ResultSet.FETCH_UNKNOWN) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - this.resultSet.setFetchDirection(direction); - } - - @Override - public int getFetchDirection() throws SQLException { - return this.resultSet.getFetchDirection(); + this.resultSet = new RestfulResultSet(database, this, resultJson); + this.affectedRows = 0; + return resultSet; } - @Override - public void setFetchSize(int rows) throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - if (rows < 0) - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - //nothing to do - } + private int executeOneUpdate(String url, String sql) throws SQLException { + if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_FOR_EXECUTE_UPDATE, "not a valid sql for executeUpdate: " + sql); - @Override - public int getFetchSize() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return 0; + String result = HttpClientPoolUtil.execute(url, sql); + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.getString("status").equals("error")) { + throw TSDBError.createSQLException(jsonObject.getInteger("code"), jsonObject.getString("desc")); + } + this.resultSet = null; + this.affectedRows = checkJsonResultSet(jsonObject); + return this.affectedRows; } - @Override - public int getResultSetConcurrency() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return this.resultSet.getConcurrency(); + private int checkJsonResultSet(JSONObject jsonObject) { + // create ... SQLs should return 0 , and Restful result is this: + // {"status": "succ", "head": ["affected_rows"], "data": [[0]], "rows": 1} + JSONArray head = jsonObject.getJSONArray("head"); + JSONArray data = jsonObject.getJSONArray("data"); + int rows = Integer.parseInt(jsonObject.getString("rows")); + if (head.size() == 1 && "affected_rows".equals(head.getString(0)) + && data.size() == 1 && data.getJSONArray(0).getInteger(0) == 0 && rows == 1) { + return 0; + } + return rows; } @Override - public int getResultSetType() throws SQLException { + public ResultSet getResultSet() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return this.resultSet.getType(); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); + return resultSet; } @Override - public void addBatch(String sql) throws SQLException { + public int getUpdateCount() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - //TODO: - } + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); - @Override - public void clearBatch() throws SQLException { - //TODO: - } - - @Override - public int[] executeBatch() throws SQLException { - //TODO: - return new int[0]; + return this.affectedRows; } @Override public Connection getConnection() throws SQLException { if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_STATEMENT_CLOSED); return this.conn; } - @Override - public boolean getMoreResults(int current) throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - if (resultSet == null) - return false; - -// switch (current) { -// case CLOSE_CURRENT_RESULT: -// resultSet.close(); -// break; -// case KEEP_CURRENT_RESULT: -// break; -// case CLOSE_ALL_RESULTS: -// resultSet.close(); -// break; -// default: -// throw new SQLException(TSDBConstants.INVALID_VARIABLES); -// } -// return next; - return false; - } - - @Override - public ResultSet getGeneratedKeys() throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public int executeUpdate(String sql, String[] columnNames) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public boolean execute(String sql, int[] columnIndexes) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public boolean execute(String sql, String[] columnNames) throws SQLException { - throw new SQLFeatureNotSupportedException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - @Override - public int getResultSetHoldability() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return this.resultSet.getHoldability(); - } - @Override public boolean isClosed() throws SQLException { return closed; } - @Override - public void setPoolable(boolean poolable) throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - //nothing to do - } - - @Override - public boolean isPoolable() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return false; - } - - @Override - public void closeOnCompletion() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - this.closeOnCompletion = true; - } - @Override - public boolean isCloseOnCompletion() throws SQLException { - if (isClosed()) - throw new SQLException(STATEMENT_CLOSED); - return this.closeOnCompletion; - } - - @Override - public T unwrap(Class iface) throws SQLException { - try { - return iface.cast(this); - } catch (ClassCastException cce) { - throw new SQLException("Unable to unwrap to " + iface.toString()); - } - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return iface.isInstance(this); - } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java similarity index 90% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java rename to src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java index 23e8796980ac419635ef6953deff1033d158962e..5e2440fd1d0e551690c5f17fe60b30c1d4a377dc 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/util/HttpClientPoolUtil.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/HttpClientPoolUtil.java @@ -1,6 +1,5 @@ -package com.taosdata.jdbc.rs.util; +package com.taosdata.jdbc.utils; -import org.apache.commons.lang3.StringUtils; import org.apache.http.HeaderElement; import org.apache.http.HeaderElementIterator; import org.apache.http.HttpEntity; @@ -23,6 +22,7 @@ import java.nio.charset.Charset; public class HttpClientPoolUtil { public static PoolingHttpClientConnectionManager cm = null; public static CloseableHttpClient httpClient = null; + public static String token = "cm9vdDp0YW9zZGF0YQ=="; /** * 默认content 类型 */ @@ -38,7 +38,7 @@ public class HttpClientPoolUtil { /** * 初始化连接池 */ - public static synchronized void initPools() { + private static synchronized void initPools() { if (httpClient == null) { cm = new PoolingHttpClientConnectionManager(); cm.setDefaultMaxPerRoute(count); @@ -50,7 +50,7 @@ public class HttpClientPoolUtil { /** * Http connection keepAlive 设置 */ - public static ConnectionKeepAliveStrategy defaultStrategy = (response, context) -> { + private static ConnectionKeepAliveStrategy defaultStrategy = (response, context) -> { HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE)); int keepTime = Http_Default_Keep_Time * 1000; while (it.hasNext()) { @@ -61,23 +61,13 @@ public class HttpClientPoolUtil { try { return Long.parseLong(value) * 1000; } catch (Exception e) { - new Exception( - "format KeepAlive timeout exception, exception:" + e.toString()) - .printStackTrace(); + new Exception("format KeepAlive timeout exception, exception:" + e.toString()).printStackTrace(); } } } return keepTime; }; - public static CloseableHttpClient getHttpClient() { - return httpClient; - } - - public static PoolingHttpClientConnectionManager getHttpConnectionManager() { - return cm; - } - /** * 执行http post请求 * 默认采用Content-Type:application/json,Accept:application/json @@ -96,8 +86,10 @@ public class HttpClientPoolUtil { initPools(); } method = (HttpEntityEnclosingRequestBase) getRequest(uri, HttpPost.METHOD_NAME, DEFAULT_CONTENT_TYPE, 0); - method.setHeader("Authorization", "Basic cm9vdDp0YW9zZGF0YQ=="); method.setHeader("Content-Type", "text/plain"); + method.setHeader("Connection", "keep-alive"); + method.setHeader("Authorization", "Taosd " + token); + method.setEntity(new StringEntity(data, Charset.forName("UTF-8"))); HttpContext context = HttpClientContext.create(); CloseableHttpResponse httpResponse = httpClient.execute(method, context); @@ -132,7 +124,7 @@ public class HttpClientPoolUtil { * @return HttpRequestBase 返回类型 * @author lisc */ - public static HttpRequestBase getRequest(String uri, String methodName, String contentType, int timeout) { + private static HttpRequestBase getRequest(String uri, String methodName, String contentType, int timeout) { if (httpClient == null) { initPools(); } @@ -153,7 +145,7 @@ public class HttpClientPoolUtil { method = new HttpPost(uri); } - if (StringUtils.isBlank(contentType)) { + if (contentType == null || contentType.isEmpty() || contentType.replaceAll("\\s", "").isEmpty()) { contentType = DEFAULT_CONTENT_TYPE; } method.addHeader("Content-Type", contentType); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..a67b4763f99164a58439df25d944cac913ab36d9 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java @@ -0,0 +1,17 @@ +package com.taosdata.jdbc.utils; + +public class OSUtils { + private static final String OS = System.getProperty("os.name").toLowerCase(); + + public static boolean isWindows() { + return OS.indexOf("win") >= 0; + } + + public static boolean isMac() { + return OS.indexOf("mac") >= 0; + } + + public static boolean isLinux() { + return OS.indexOf("nux") >= 0; + } +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java index f0d92346167411ca6da78015392ba8a21286e1cf..251ca2af013e2b1c9cb314b776621455c91d9384 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java @@ -20,8 +20,11 @@ import java.sql.Connection; public class SqlSyntaxValidator { - private static final String[] updateSQL = {"insert", "update", "delete", "create", "alter", "drop", "show", "describe", "use", "import"}; - private static final String[] querySQL = {"select"}; + private static final String[] SQL = {"select", "insert", "import", "create", "use", "alter", "drop", "set", "show", "describe"}; + private static final String[] updateSQL = {"insert", "import", "create", "use", "alter", "drop", "set"}; + private static final String[] querySQL = {"select", "show", "describe"}; + + private static final String[] databaseUnspecifiedShow = {"databases", "dnodes", "mnodes", "variables"}; private TSDBConnection tsdbConnection; @@ -37,8 +40,38 @@ public class SqlSyntaxValidator { return false; } + public static boolean isValidForExecuteQuery(String sql) { + for (String prefix : querySQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; + } + return false; + } + + public static boolean isValidForExecute(String sql) { + for (String prefix : SQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; + } + return false; + } + + public static boolean isDatabaseUnspecifiedQuery(String sql) { + for (String databaseObj : databaseUnspecifiedShow) { + if (sql.trim().toLowerCase().matches("show\\s+" + databaseObj + ".*")) + return true; + } + return false; + } + + public static boolean isDatabaseUnspecifiedUpdate(String sql) { + sql = sql.trim().toLowerCase(); + return sql.matches("create\\s+database.*") || sql.startsWith("set") || sql.matches("drop\\s+database.*"); + } + public static boolean isUseSql(String sql) { - return sql.trim().toLowerCase().startsWith("use") || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); + return sql.trim().toLowerCase().startsWith("use"); +// || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); } public static boolean isShowSql(String sql) { @@ -58,8 +91,9 @@ public class SqlSyntaxValidator { return sql.trim().toLowerCase().startsWith("select"); } - public static boolean isShowDatabaseSql(String sql) { return sql.trim().toLowerCase().matches("show\\s*databases"); } + + } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java deleted file mode 100644 index 800265868da6da5047453d542fc9e37c69fe1e36..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java +++ /dev/null @@ -1,272 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; -import java.util.*; -import java.util.concurrent.TimeUnit; - -public class TDNode { - - private int index; - private int running; - private int deployed; - private boolean testCluster; - private String path; - private String cfgDir; - private String dataDir; - private String logDir; - private String cfgPath; - - public TDNode(int index) { - this.index = index; - running = 0; - deployed = 0; - testCluster = false; - } - - public void setPath(String path) { - this.path = path; - } - - public void setTestCluster(boolean testCluster) { - this.testCluster = testCluster; - } - - public void setRunning(int running) { - this.running = running; - } - - public void searchTaosd(File dir, ArrayList taosdPath) { - File[] fileList = dir.listFiles(); - - if(fileList == null || fileList.length == 0) { - return; - } - - for(File file : fileList) { - if(file.isFile()) { - if(file.getName().equals("taosd")) { - taosdPath.add(file.getAbsolutePath()); - } - } else { - searchTaosd(file, taosdPath); - } - } - } - - public void start() { - String selfPath = System.getProperty("user.dir"); - String binPath = ""; - String projDir = selfPath + "/../../../"; - - try { - ArrayList taosdPath = new ArrayList<>(); - - File dir = new File(projDir); - String realProjDir = dir.getCanonicalPath(); - dir = new File(realProjDir); - System.out.println("project Dir: " + projDir); - searchTaosd(dir, taosdPath); - - if(taosdPath.size() == 0) { - System.out.println("The project path doens't exist"); - return; - } else { - for(String p : taosdPath) { - if(!p.contains("packaging")) { - binPath = p; - } - } - } - } catch (Exception e) { - e.printStackTrace(); - } - - if(binPath.isEmpty()) { - System.out.println("taosd not found"); - return; - } else { - System.out.println("taosd found in " + binPath); - } - - if(this.deployed == 0) { - System.out.println("dnode" + index + "is not deployed"); - return; - } - - String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & "; - System.out.println("start taosd cmd: " + cmd); - - try{ - Runtime.getRuntime().exec(cmd); - TimeUnit.SECONDS.sleep(5); - } catch (Exception e) { - e.printStackTrace(); - } - - this.running = 1; - } - - public Integer getTaosdPid() { - String cmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"; - String[] cmds = {"sh", "-c", cmd}; - try { - Process process = Runtime.getRuntime().exec(cmds); - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line = null; - Integer res = null; - while((line = reader.readLine()) != null) { - if(!line.isEmpty()) { - res = Integer.valueOf(line); - break; - } - } - - return res; - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - public void stop() { - - if (this.running != 0) { - Integer pid = null; - while((pid = getTaosdPid()) != null) { - - String killCmd = "kill -term " + pid; - String[] killCmds = {"sh", "-c", killCmd}; - try { - Runtime.getRuntime().exec(killCmds).waitFor(); - - TimeUnit.SECONDS.sleep(2); - } catch (Exception e) { - e.printStackTrace(); - } - } - - try { - for(int port = 6030; port < 6041; port ++) { - String fuserCmd = "fuser -k -n tcp " + port; - Runtime.getRuntime().exec(fuserCmd).waitFor(); - } - } catch (Exception e) { - e.printStackTrace(); - } - - this.running = 0; - System.out.println("dnode:" + this.index + " is stopped by kill -term"); - } - } - - public void startIP() { - try{ - String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " up"; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void stopIP() { - try{ - String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down"; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public void setCfgConfig(String option, String value) { - try{ - String cmd = "echo " + option + " " + value + " >> " + this.cfgPath; - String[] cmdLine = {"sh", "-c", cmd}; - Process ps = Runtime.getRuntime().exec(cmdLine); - ps.waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public String getDnodeRootDir() { - String dnodeRootDir = this.path + "/sim/psim/dnode" + this.index; - return dnodeRootDir; - } - - public String getDnodesRootDir() { - String dnodesRootDir = this.path + "/sim/psim" + this.index; - return dnodesRootDir; - } - - public void deploy() { - this.logDir = this.path + "/sim/dnode" + this.index + "/log"; - this.dataDir = this.path + "/sim/dnode" + this.index + "/data"; - this.cfgDir = this.path + "/sim/dnode" + this.index + "/cfg"; - this.cfgPath = this.path + "/sim/dnode" + this.index + "/cfg/taos.cfg"; - - try { - String cmd = "rm -rf " + this.logDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "rm -rf " + this.cfgDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "rm -rf " + this.dataDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.logDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.cfgDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "mkdir -p " + this.dataDir; - Runtime.getRuntime().exec(cmd).waitFor(); - - cmd = "touch " + this.cfgPath; - Runtime.getRuntime().exec(cmd).waitFor(); - } catch (Exception e) { - e.printStackTrace(); - } - - if(this.testCluster) { - startIP(); - setCfgConfig("masterIp", "192.168.0.1"); - setCfgConfig("secondIp", "192.168.0.2"); - setCfgConfig("publicIp", "192.168.0." + this.index); - setCfgConfig("internalIp", "192.168.0." + this.index); - setCfgConfig("privateIp", "192.168.0." + this.index); - } - setCfgConfig("dataDir", this.dataDir); - setCfgConfig("logDir", this.logDir); - setCfgConfig("numOfLogLines", "1000000/00"); - setCfgConfig("mnodeEqualVnodeNum", "0"); - setCfgConfig("walLevel", "1"); - setCfgConfig("statusInterval", "1"); - setCfgConfig("numOfMnodes", "3"); - setCfgConfig("numOfThreadsPerCore", "2.0"); - setCfgConfig("monitor", "0"); - setCfgConfig("maxVnodeConnections", "30000"); - setCfgConfig("maxMgmtConnections", "30000"); - setCfgConfig("maxMeterConnections", "30000"); - setCfgConfig("maxShellConns", "30000"); - setCfgConfig("locale", "en_US.UTF-8"); - setCfgConfig("charset", "UTF-8"); - setCfgConfig("asyncLog", "0"); - setCfgConfig("anyIp", "0"); - setCfgConfig("dDebugFlag", "135"); - setCfgConfig("mDebugFlag", "135"); - setCfgConfig("sdbDebugFlag", "135"); - setCfgConfig("rpcDebugFlag", "135"); - setCfgConfig("tmrDebugFlag", "131"); - setCfgConfig("cDebugFlag", "135"); - setCfgConfig("httpDebugFlag", "135"); - setCfgConfig("monitorDebugFlag", "135"); - setCfgConfig("udebugFlag", "135"); - setCfgConfig("jnidebugFlag", "135"); - setCfgConfig("qdebugFlag", "135"); - this.deployed = 1; - } -} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java deleted file mode 100644 index efc4c53e2864789811ca30110f9b69e5f685a89e..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.taosdata.jdbc.utils; - -import java.io.File; -import java.util.*; - -public class TDNodes { - private ArrayList tdNodes; - private boolean testCluster; - - public TDNodes () { - tdNodes = new ArrayList<>(); - for(int i = 1; i < 11; i ++) { - tdNodes.add(new TDNode(i)); - } - } - - public void setTestCluster(boolean testCluster) { - this.testCluster = testCluster; - } - - public void check(int index) { - if(index < 1 || index > 10) { - System.out.println("index: " + index + " should on a scale of [1, 10]"); - return; - } - } - - public void deploy(int index) { - try { - File file = new File(System.getProperty("user.dir") + "/../../../"); - String projectRealPath = file.getCanonicalPath(); - check(index); - tdNodes.get(index - 1).setTestCluster(this.testCluster); - tdNodes.get(index - 1).setPath(projectRealPath); - tdNodes.get(index - 1).deploy(); - } catch (Exception e) { - e.printStackTrace(); - System.out.println("deploy Test Exception"); - } - } - - public void cfg(int index, String option, String value) { - check(index); - tdNodes.get(index - 1).setCfgConfig(option, value); - } - - public TDNode getTDNode(int index) { - check(index); - return tdNodes.get(index - 1); - } - - public void start(int index) { - check(index); - tdNodes.get(index - 1).start(); - } - - public void stop(int index) { - check(index); - tdNodes.get(index - 1).stop(); - } - - public void startIP(int index) { - check(index); - tdNodes.get(index - 1).startIP(); - } - - public void stopIP(int index) { - check(index); - tdNodes.get(index - 1).stopIP(); - } - -} \ No newline at end of file diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java new file mode 100644 index 0000000000000000000000000000000000000000..ee1364ce217c639cbf864f87f83ad6f5ddd6c137 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java @@ -0,0 +1,74 @@ +package com.taosdata.jdbc.utils; + +import javax.management.*; +import java.lang.management.ManagementFactory; +import java.util.concurrent.atomic.AtomicLong; + +public class TaosInfo implements TaosInfoMBean { + + private static volatile TaosInfo instance; + private AtomicLong connect_open = new AtomicLong(); + private AtomicLong connect_close = new AtomicLong(); + private AtomicLong statement_count = new AtomicLong(); + + static { + try { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + ObjectName name = new ObjectName("TaosInfoMBean:name=TaosInfo"); + server.registerMBean(TaosInfo.getInstance(), name); + + } catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException e) { + e.printStackTrace(); + } + } + + @Override + public long getConnect_open() { + return connect_open.get(); + } + + @Override + public long getConnect_close() { + return connect_close.get(); + } + + @Override + public long getConnect_active() { + return connect_open.get() - connect_close.get(); + } + + @Override + public long getStatement_count() { + return statement_count.get(); + } + + /*******************************************************/ + + public void conn_open_increment() { + connect_open.incrementAndGet(); + } + + public void connect_close_increment() { + connect_close.incrementAndGet(); + } + + public void stmt_count_increment() { + statement_count.incrementAndGet(); + } + + /********************************************************************************/ + private TaosInfo() { + } + + public static TaosInfo getInstance() { + if (instance == null) { + synchronized (TaosInfo.class) { + if (instance == null) { + instance = new TaosInfo(); + } + } + } + return instance; + } + +} diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java new file mode 100644 index 0000000000000000000000000000000000000000..e16f41b2f585244721e923a963c53fb6b7bccf47 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java @@ -0,0 +1,13 @@ +package com.taosdata.jdbc.utils; + +public interface TaosInfoMBean { + + long getConnect_open(); + + long getConnect_close(); + + long getConnect_active(); + + long getStatement_count(); + +} diff --git a/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver b/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver index e65b4e3b2215df049ef4a020f1aac3bb45d60b61..893f7cdf34c9934ffb9db91328967995d406b19b 100644 --- a/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver +++ b/src/connector/jdbc/src/main/resources/META-INF/services/java.sql.Driver @@ -1 +1,2 @@ com.taosdata.jdbc.TSDBDriver +com.taosdata.jdbc.rs.RestfulDriver diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java deleted file mode 100644 index 1edb957493183c407b668bd68532e83fa1a64dba..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/TestPreparedStatement.java +++ /dev/null @@ -1,36 +0,0 @@ -import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.TSDBPreparedStatement; - -import java.sql.*; -import java.util.Properties; - -public class TestPreparedStatement { - - public static void main(String[] args) throws SQLException { - Connection connection = null; - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); - String rawSql = "select * from test.log0601"; -// String[] params = new String[]{"ts", "c1"}; - PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql); - ResultSet resSet = pstmt.executeQuery(); - while(resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s \n", i, resSet.getString(i)); - } - } - resSet.close(); - pstmt.close(); - connection.close(); - - } catch (Exception e) { - e.printStackTrace(); - if (null != connection) { - connection.close(); - } - } - } -} diff --git a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java deleted file mode 100644 index 39a08f0fe98a279e62ca07f20047813edeab56db..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java +++ /dev/null @@ -1,33 +0,0 @@ -import com.taosdata.jdbc.TSDBDriver; - -import java.sql.*; -import java.util.Properties; - -public class TestTSDBDatabaseMetaData { - - public static void main(String[] args) throws SQLException { - Connection connection = null; - DatabaseMetaData dbMetaData = null; - ResultSet resSet = null; - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); - dbMetaData = connection.getMetaData(); - resSet = dbMetaData.getCatalogs(); - while(resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf("dbMetaData.getCatalogs(%d) = %s\n", i, resSet.getString(i)); - } - } - resSet.close(); - - } catch (Exception e) { - e.printStackTrace(); - if (null != connection) { - connection.close(); - } - } - } -} diff --git a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java deleted file mode 100644 index 47acb200649be4c8c6af5817c2ef4c074a2a3fa0..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java +++ /dev/null @@ -1,78 +0,0 @@ -import com.taosdata.jdbc.TSDBConnection; -import com.taosdata.jdbc.TSDBDriver; -import com.taosdata.jdbc.TSDBResultSet; -import com.taosdata.jdbc.TSDBSubscribe; - -import java.sql.DriverManager; -import java.util.Properties; - -public class TestTSDBSubscribe { - - public static TSDBConnection connectTDengine(String host, String database) throws Exception { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - - String cs = String.format("jdbc:TAOS://%s:0/%s", host, database); - return (TSDBConnection)DriverManager.getConnection(cs, properties); - } - - public static void main(String[] args) throws Exception { - String usage = "java -Djava.ext.dirs=../ TestTSDBSubscribe [-host host] <-db database> <-topic topic> <-sql sql>"; - if (args.length < 2) { - System.err.println(usage); - return; - } - - String host = "localhost", database = "", topic = "", sql = ""; - for (int i = 0; i < args.length; i++) { - if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) { - database = args[++i]; - } - if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { - topic = args[++i]; - } - if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { - host = args[++i]; - } - if ("-sql".equalsIgnoreCase(args[i]) && i < args.length - 1) { - sql = args[++i]; - } - } - if (database.isEmpty() || topic.isEmpty() || sql.isEmpty()) { - System.err.println(usage); - return; - } - - TSDBConnection connection = null; - TSDBSubscribe sub = null; - try { - connection = connectTDengine(host, database); - sub = ((TSDBConnection) connection).subscribe(topic, sql, false); - - int total = 0; - while(true) { - TSDBResultSet rs = sub.consume(); - int count = 0; - while(rs.next()) { - count++; - } - total += count; - System.out.printf("%d rows consumed, total %d\n", count, total); - Thread.sleep(900); - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - if (null != sub) { - sub.close(true); - } - if (null != connection) { - connection.close(); - } - } - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java deleted file mode 100644 index ce3735c12894807efadd1f5673fc34eee43ae01b..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.taosdata.jdbc; - -import com.taosdata.jdbc.utils.TDNodes; - -import org.junit.AfterClass; -import org.junit.BeforeClass; - -public abstract class BaseTest { - - private static boolean testCluster = false; - private static TDNodes nodes = new TDNodes(); - - @BeforeClass - public static void setupEnv() { - try { - if (nodes.getTDNode(1).getTaosdPid() != null) { - System.out.println("Kill taosd before running JDBC test"); - nodes.getTDNode(1).setRunning(1); - nodes.stop(1); - } - nodes.setTestCluster(testCluster); - nodes.deploy(1); - nodes.start(1); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @AfterClass - public static void cleanUpEnv() { - nodes.stop(1); - } -} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java deleted file mode 100644 index 7d96cbb5381d099f7b67ecdb5414093c54b37b65..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java +++ /dev/null @@ -1,120 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; - -public class BatchInsertTest extends BaseTest { - - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String stbName = "meters"; - static String host = "localhost"; - static int numOfTables = 30; - final static int numOfRecordsPerTable = 1000; - static long ts = 1496732686000l; - final static String tablePrefix = "t"; - - @Before - public void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("use " + dbName); - - String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; - statement.executeUpdate(createTableSql); - - for(int i = 0; i < numOfTables; i++) { - String loc = i % 2 == 0 ? "beijing" : "shanghai"; - String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; - statement.executeUpdate(createSubTalbesSql); - } - - } - - @Test - public void testBatchInsert() throws SQLException{ - - ExecutorService executorService = Executors.newFixedThreadPool(numOfTables); - - for (int i = 0; i < numOfTables; i++) { - final int index = i; - executorService.execute(new Runnable() { - @Override - public void run() { - try { - long startTime = System.currentTimeMillis(); - Statement statement = connection.createStatement(); // get statement - StringBuilder sb = new StringBuilder(); - sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); - Random rand = new Random(); - for (int j = 1; j <= numOfRecordsPerTable; j++) { - sb.append("(" + (ts + j) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ")"); - } - statement.addBatch(sb.toString()); - statement.executeBatch(); - long endTime = System.currentTimeMillis(); - System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); - connection.commit(); - statement.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - }); - } - - executorService.shutdown(); - - try { - executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - Statement statement = connection.createStatement(); - ResultSet rs = statement.executeQuery("select * from meters"); - int num = 0; - while (rs.next()) { - num++; - } - assertEquals(num, numOfTables * numOfRecordsPerTable); - rs.close(); - } - - - @After - public void close() throws Exception { - statement.close(); - connection.close(); - Thread.sleep(10); - } - -} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java deleted file mode 100644 index a54ece4eadc54e930a4af4faddb58dfef3ceb25a..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.Test; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Properties; - -import static org.junit.Assert.assertTrue; - -public class ConnectionTest extends BaseTest { - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String stbName = "st"; - static String host = "localhost"; - - @Test - public void testConnection() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - assertTrue(null != connection); - statement = connection.createStatement(); - assertTrue(null != statement); - - // try reconnect - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - try { - statement.execute("create database if not exists " + dbName); - } catch (SQLException e) { - assert false : "create database error: " + e.getMessage(); - } - - try { - if (!connection.isClosed()) { - if (!statement.isClosed()) { - statement.executeUpdate("drop database " + dbName); - statement.close(); - } - connection.close(); - Thread.sleep(10); - } - } catch (Exception e) { - assert false : "close connection error: " + e.getMessage(); - } - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java deleted file mode 100644 index 19dabe07462b6481b2ab5460aad7ac5712e21b7c..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java +++ /dev/null @@ -1,245 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; - -public class DatabaseMetaDataTest { - static Connection connection = null; - static PreparedStatement statement = null; - static String dbName = "test"; - static String tName = "t0"; - static String host = "localhost"; - - @BeforeClass - public static void createConnection() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - String sql = "drop database if exists " + dbName; - statement = connection.prepareStatement(sql); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - } - - @Test - public void testMetaDataTest() throws SQLException { - DatabaseMetaData databaseMetaData = connection.getMetaData(); - ResultSet resultSet = databaseMetaData.getTables(dbName, "t*", "t*", new String[]{"t"}); - while (resultSet.next()) { - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s\n", i, resultSet.getString(i)); - } - } - resultSet.close(); - databaseMetaData.isWrapperFor(null); - databaseMetaData.allProceduresAreCallable(); - databaseMetaData.allTablesAreSelectable(); - databaseMetaData.getURL(); - databaseMetaData.getUserName(); - databaseMetaData.isReadOnly(); - databaseMetaData.nullsAreSortedHigh(); - databaseMetaData.nullsAreSortedLow(); - databaseMetaData.nullsAreSortedAtStart(); - databaseMetaData.nullsAreSortedAtEnd(); - databaseMetaData.getDatabaseProductName(); - databaseMetaData.getDatabaseProductVersion(); - databaseMetaData.getDriverName(); - databaseMetaData.getDriverVersion(); - databaseMetaData.getDriverMajorVersion(); - databaseMetaData.getDriverMinorVersion(); - databaseMetaData.usesLocalFiles(); - databaseMetaData.usesLocalFilePerTable(); - databaseMetaData.supportsMixedCaseIdentifiers(); - databaseMetaData.storesUpperCaseIdentifiers(); - databaseMetaData.storesLowerCaseIdentifiers(); - databaseMetaData.storesMixedCaseIdentifiers(); - databaseMetaData.supportsMixedCaseQuotedIdentifiers(); - databaseMetaData.storesUpperCaseQuotedIdentifiers(); - databaseMetaData.storesLowerCaseQuotedIdentifiers(); - databaseMetaData.storesMixedCaseQuotedIdentifiers(); - databaseMetaData.getIdentifierQuoteString(); - databaseMetaData.getSQLKeywords(); - databaseMetaData.getNumericFunctions(); - databaseMetaData.getStringFunctions(); - databaseMetaData.getSystemFunctions(); - databaseMetaData.getTimeDateFunctions(); - databaseMetaData.getSearchStringEscape(); - databaseMetaData.getExtraNameCharacters(); - databaseMetaData.supportsAlterTableWithAddColumn(); - databaseMetaData.supportsAlterTableWithDropColumn(); - databaseMetaData.supportsColumnAliasing(); - databaseMetaData.nullPlusNonNullIsNull(); - databaseMetaData.supportsConvert(); - databaseMetaData.supportsConvert(0, 0); - databaseMetaData.supportsTableCorrelationNames(); - databaseMetaData.supportsDifferentTableCorrelationNames(); - databaseMetaData.supportsExpressionsInOrderBy(); - databaseMetaData.supportsOrderByUnrelated(); - databaseMetaData.supportsGroupBy(); - databaseMetaData.supportsGroupByUnrelated(); - databaseMetaData.supportsGroupByBeyondSelect(); - databaseMetaData.supportsLikeEscapeClause(); - databaseMetaData.supportsMultipleResultSets(); - databaseMetaData.supportsMultipleTransactions(); - databaseMetaData.supportsNonNullableColumns(); - databaseMetaData.supportsMinimumSQLGrammar(); - databaseMetaData.supportsCoreSQLGrammar(); - databaseMetaData.supportsExtendedSQLGrammar(); - databaseMetaData.supportsANSI92EntryLevelSQL(); - databaseMetaData.supportsANSI92IntermediateSQL(); - databaseMetaData.supportsANSI92FullSQL(); - databaseMetaData.supportsIntegrityEnhancementFacility(); - databaseMetaData.supportsOuterJoins(); - databaseMetaData.supportsFullOuterJoins(); - databaseMetaData.supportsLimitedOuterJoins(); - databaseMetaData.getSchemaTerm(); - databaseMetaData.getProcedureTerm(); - databaseMetaData.getCatalogTerm(); - databaseMetaData.isCatalogAtStart(); - databaseMetaData.getCatalogSeparator(); - databaseMetaData.supportsSchemasInDataManipulation(); - databaseMetaData.supportsSchemasInProcedureCalls(); - databaseMetaData.supportsSchemasInTableDefinitions(); - databaseMetaData.supportsSchemasInIndexDefinitions(); - databaseMetaData.supportsSchemasInPrivilegeDefinitions(); - databaseMetaData.supportsCatalogsInDataManipulation(); - databaseMetaData.supportsCatalogsInProcedureCalls(); - databaseMetaData.supportsCatalogsInTableDefinitions(); - databaseMetaData.supportsCatalogsInIndexDefinitions(); - databaseMetaData.supportsCatalogsInPrivilegeDefinitions(); - databaseMetaData.supportsPositionedDelete(); - databaseMetaData.supportsPositionedUpdate(); - databaseMetaData.supportsSelectForUpdate(); - databaseMetaData.supportsStoredProcedures(); - databaseMetaData.supportsSubqueriesInComparisons(); - databaseMetaData.supportsSubqueriesInExists(); - databaseMetaData.supportsSubqueriesInIns(); - databaseMetaData.supportsSubqueriesInQuantifieds(); - databaseMetaData.supportsCorrelatedSubqueries(); - databaseMetaData.supportsUnion(); - databaseMetaData.supportsUnionAll(); - databaseMetaData.supportsOpenCursorsAcrossCommit(); - databaseMetaData.supportsOpenCursorsAcrossRollback(); - databaseMetaData.supportsOpenStatementsAcrossCommit(); - databaseMetaData.supportsOpenStatementsAcrossRollback(); - databaseMetaData.getMaxBinaryLiteralLength(); - databaseMetaData.getMaxCharLiteralLength(); - databaseMetaData.getMaxColumnNameLength(); - databaseMetaData.getMaxColumnsInGroupBy(); - databaseMetaData.getMaxColumnsInIndex(); - databaseMetaData.getMaxColumnsInOrderBy(); - databaseMetaData.getMaxColumnsInSelect(); - databaseMetaData.getMaxColumnsInTable(); - databaseMetaData.getMaxConnections(); - databaseMetaData.getMaxCursorNameLength(); - databaseMetaData.getMaxIndexLength(); - databaseMetaData.getMaxSchemaNameLength(); - databaseMetaData.getMaxProcedureNameLength(); - databaseMetaData.getMaxCatalogNameLength(); - databaseMetaData.getMaxRowSize(); - databaseMetaData.doesMaxRowSizeIncludeBlobs(); - databaseMetaData.getMaxStatementLength(); - databaseMetaData.getMaxStatements(); - databaseMetaData.getMaxTableNameLength(); - databaseMetaData.getMaxTablesInSelect(); - databaseMetaData.getMaxUserNameLength(); - databaseMetaData.getDefaultTransactionIsolation(); - databaseMetaData.supportsTransactions(); - databaseMetaData.supportsTransactionIsolationLevel(0); - databaseMetaData.supportsDataDefinitionAndDataManipulationTransactions(); - databaseMetaData.supportsDataManipulationTransactionsOnly(); - databaseMetaData.dataDefinitionCausesTransactionCommit(); - databaseMetaData.dataDefinitionIgnoredInTransactions(); - try { - databaseMetaData.getProcedures("", "", ""); - } catch (Exception e) { - } - try { - databaseMetaData.getProcedureColumns("", "", "", ""); - } catch (Exception e) { - } - try { - databaseMetaData.getTables("", "", "", new String[]{""}); - } catch (Exception e) { - } - databaseMetaData.getSchemas(); - databaseMetaData.getCatalogs(); -// databaseMetaData.getTableTypes(); - - databaseMetaData.getColumns(dbName, "", tName, ""); - databaseMetaData.getColumnPrivileges("", "", "", ""); - databaseMetaData.getTablePrivileges("", "", ""); - databaseMetaData.getBestRowIdentifier("", "", "", 0, false); - databaseMetaData.getVersionColumns("", "", ""); - databaseMetaData.getPrimaryKeys("", "", ""); - databaseMetaData.getImportedKeys("", "", ""); - databaseMetaData.getExportedKeys("", "", ""); - databaseMetaData.getCrossReference("", "", "", "", "", ""); - databaseMetaData.getTypeInfo(); - databaseMetaData.getIndexInfo("", "", "", false, false); - databaseMetaData.supportsResultSetType(0); - databaseMetaData.supportsResultSetConcurrency(0, 0); - databaseMetaData.ownUpdatesAreVisible(0); - databaseMetaData.ownDeletesAreVisible(0); - databaseMetaData.ownInsertsAreVisible(0); - databaseMetaData.othersUpdatesAreVisible(0); - databaseMetaData.othersDeletesAreVisible(0); - databaseMetaData.othersInsertsAreVisible(0); - databaseMetaData.updatesAreDetected(0); - databaseMetaData.deletesAreDetected(0); - databaseMetaData.insertsAreDetected(0); - databaseMetaData.supportsBatchUpdates(); - databaseMetaData.getUDTs("", "", "", new int[]{0}); - databaseMetaData.getConnection(); - databaseMetaData.supportsSavepoints(); - databaseMetaData.supportsNamedParameters(); - databaseMetaData.supportsMultipleOpenResults(); - databaseMetaData.supportsGetGeneratedKeys(); - databaseMetaData.getSuperTypes("", "", ""); - databaseMetaData.getSuperTables("", "", ""); - databaseMetaData.getAttributes("", "", "", ""); - databaseMetaData.supportsResultSetHoldability(0); - databaseMetaData.getResultSetHoldability(); - databaseMetaData.getDatabaseMajorVersion(); - databaseMetaData.getDatabaseMinorVersion(); - databaseMetaData.getJDBCMajorVersion(); - databaseMetaData.getJDBCMinorVersion(); - databaseMetaData.getSQLStateType(); - databaseMetaData.locatorsUpdateCopy(); - databaseMetaData.supportsStatementPooling(); - databaseMetaData.getRowIdLifetime(); - databaseMetaData.getSchemas("", ""); - databaseMetaData.supportsStoredFunctionsUsingCallSyntax(); - databaseMetaData.autoCommitFailureClosesAllResultSets(); - databaseMetaData.getClientInfoProperties(); - databaseMetaData.getFunctions("", "", ""); - databaseMetaData.getFunctionColumns("", "", "", ""); - databaseMetaData.getPseudoColumns("", "", "", ""); - databaseMetaData.generatedKeyAlwaysReturned(); - - } - - @AfterClass - public static void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java deleted file mode 100644 index dbe16d9fea33be117cfa93a3ecd9a1ff4bd12cd8..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; - -public class ImportTest extends BaseTest { - Connection connection = null; - Statement statement = null; - String dbName = "test"; - String tName = "t0"; - String host = "localhost"; - - @Before - public void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - - } - - @Test - public void insertData() throws Exception { - long ts = 1496732686000l; - - for (int i = 0; i < 50; i++) { - ts++; - int row = statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); - System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); - assertEquals(1, row); - } - } - - @Test - public void selectData() throws Exception { - insertData(); - String sql = "select * from test.t0"; - ResultSet resSet = statement.executeQuery(sql); - - while (resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf(i + ": " + resSet.getString(i) + "\t"); - } - } - resSet.close(); - } - - @Test - public void importData() throws Exception { - // 避免时间重复 - long ts = 1496732686000l; - - StringBuilder sqlBuilder = new StringBuilder("insert into ").append(dbName).append(".").append(tName).append(" values "); - - for (int i = 0; i < 50; i++) { - int a = i / 5; - long t = ts + a; - sqlBuilder.append("(").append(t).append(",").append((100 + i)).append(",").append(i).append(") "); - } - System.out.println(sqlBuilder.toString()); - int rows = statement.executeUpdate(sqlBuilder.toString()); - System.out.println(rows); - assertEquals(10, rows); - } - - @After - public void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java deleted file mode 100644 index 0535214ac198685ad65edf338bc6d72f841731a8..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java +++ /dev/null @@ -1,200 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -import java.sql.*; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@FixMethodOrder(MethodSorters.DEFAULT) -public class PreparedStatementTest extends BaseTest { - static Connection connection = null; - static PreparedStatement statement = null; - static String dbName = "test"; - static String tName = "t0"; - static String host = "localhost"; - static ResultSet resSet = null; - - @BeforeClass - public static void createConnection() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - String sql = "drop database if exists " + dbName; - statement = (TSDBPreparedStatement) connection.prepareStatement(sql); - - } - - @Test - public void createTableAndQuery() throws SQLException { - long ts = System.currentTimeMillis(); - - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + "(ts timestamp, k1 int)"); - statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); - - PreparedStatement selectStatement = connection.prepareStatement("select * from " + dbName + "." + tName); - - ResultSet resultSet = selectStatement.executeQuery(); - assertTrue(null != resultSet); - - boolean isClosed = statement.isClosed(); - assertEquals(false, isClosed); - } - - @Test - public void testPreparedStatement() throws SQLException { - long ts = System.currentTimeMillis() + 20000; - PreparedStatement saveStatement = connection - .prepareStatement("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); - - int affectedRows = saveStatement.executeUpdate(); - assertTrue(1 == affectedRows); - } - - @Test - public void testSavedPreparedStatement() throws SQLException { - long ts = System.currentTimeMillis(); - - TSDBPreparedStatement saveStatement = (TSDBPreparedStatement) connection - .prepareStatement("insert into " + dbName + "." + tName + " values (?, ?)"); - - saveStatement.setObject(1, ts + 10000); - saveStatement.setObject(2, 3); - int rows = saveStatement.executeUpdate(); - assertEquals(1, rows); - } - - @Test - public void testUnsupport() { - // if(null == resSet) { - // return; - // } - TSDBPreparedStatement tsdbStatement = (TSDBPreparedStatement) statement; - try { - tsdbStatement.unwrap(null); - } catch (SQLException e) { - } - try { - tsdbStatement.isWrapperFor(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMaxFieldSize(); - } catch (SQLException e) { - } - try { - tsdbStatement.setMaxFieldSize(0); - } catch (SQLException e) { - } - try { - tsdbStatement.setEscapeProcessing(true); - } catch (SQLException e) { - } - try { - tsdbStatement.cancel(); - } catch (SQLException e) { - } - try { - tsdbStatement.getWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.clearWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.setCursorName(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.setFetchDirection(0); - } catch (SQLException e) { - } - try { - tsdbStatement.getFetchDirection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetConcurrency(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetType(); - } catch (SQLException e) { - } - try { - tsdbStatement.getConnection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.getGeneratedKeys(); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, 0); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new int[]{0}); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new String[]{"str1", "str2"}); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetHoldability(); - } catch (SQLException e) { - } - try { - tsdbStatement.setPoolable(true); - } catch (SQLException e) { - } - try { - tsdbStatement.isPoolable(); - } catch (SQLException e) { - } - try { - tsdbStatement.closeOnCompletion(); - } catch (SQLException e) { - } - try { - tsdbStatement.isCloseOnCompletion(); - } catch (SQLException e) { - } - } - - @AfterClass - public static void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } - -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java deleted file mode 100644 index 611e21e887b1988a66019d437133d6c3f7926961..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/QueryDataTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; -import java.util.Random; - -import static org.junit.Assert.assertEquals; -import java.util.Properties; -import java.util.concurrent.Executors; -import java.util.concurrent.*; - -import static org.junit.Assert.assertTrue; - -public class QueryDataTest extends BaseTest { - - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String stbName = "meters"; - static String host = "localhost"; - static int numOfTables = 30; - final static int numOfRecordsPerTable = 1000; - static long ts = 1496732686000l; - final static String tablePrefix = "t"; - - @Before - public void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("use " + dbName); - - String createTableSql = "create table " + stbName + "(ts timestamp, name binary(6))"; - statement.executeUpdate(createTableSql); - } - - @Test - public void testQueryBinaryData() throws SQLException{ - - String insertSql = "insert into " + stbName + " values(now, 'taosda')"; - System.out.println(insertSql); - - statement.executeUpdate(insertSql); - - String querySql = "select * from " + stbName; - ResultSet rs = statement.executeQuery(querySql); - - while(rs.next()) { - String name = rs.getString(2) + "001"; - System.out.println("name = " + name); - assertEquals(name, "taosda001"); - } - rs.close(); - } - - - @After - public void close() throws Exception { - statement.close(); - connection.close(); - Thread.sleep(10); - } - -} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java deleted file mode 100644 index 8067c547df148587766d87e204f5abad2bfb659d..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ /dev/null @@ -1,827 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import javax.sql.rowset.serial.SerialBlob; -import javax.sql.rowset.serial.SerialClob; -import java.sql.*; -import java.util.HashMap; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class ResultSetTest extends BaseTest { - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String tName = "t0"; - static String host = "localhost"; - static ResultSet resSet = null; - - @BeforeClass - public static void createDatabaseAndTable() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + - " (ts timestamp, k1 int, k2 bigint, k3 float, k4 double, k5 binary(30), k6 smallint, k7 bool, k8 nchar(20))"); - - statement.executeQuery("use " + dbName); - } - - @Test - public void testResultSet() { - String sql = null; - long ts = 1496732686000l; - int v1 = 2147483600; - long v2 = ts + 1000; - float v3 = 3.1415926f; - double v4 = 3.1415926535897; - String v5 = "涛思数据,强~!"; - short v6 = 12; - boolean v7 = false; - String v8 = "TDengine is powerful"; - - sql = "insert into " + dbName + "." + tName + " values (" + ts + "," + v1 + "," + v2 + "," + v3 + "," + v4 - + ",\"" + v5 + "\"," + v6 + "," + v7 + ",\"" + v8 + "\")"; - - try { - statement.executeUpdate(sql); - assertEquals(1, statement.getUpdateCount()); - } catch (SQLException e) { - assert false : "insert error " + e.getMessage(); - } - - try { - statement.executeQuery("select * from " + dbName + "." + tName + " where ts = " + ts); - resSet = statement.getResultSet(); - System.out.println(((TSDBResultSet) resSet).getRowData()); - while (resSet.next()) { - assertEquals(ts, resSet.getLong(1)); - assertEquals(ts, resSet.getLong("ts")); - - System.out.println(resSet.getTimestamp(1)); - - assertEquals(v1, resSet.getInt(2)); - assertEquals(v1, resSet.getInt("k1")); - - assertEquals(v2, resSet.getLong(3)); - assertEquals(v2, resSet.getLong("k2")); - - assertEquals(v3, resSet.getFloat(4), 7); - assertEquals(v3, resSet.getFloat("k3"), 7); - - assertEquals(v4, resSet.getDouble(5), 13); - assertEquals(v4, resSet.getDouble("k4"), 13); - - assertEquals(v5, resSet.getString(6)); - assertEquals(v5, resSet.getString("k5")); - - assertEquals(v6, resSet.getShort(7)); - assertEquals(v6, resSet.getShort("k6")); - - assertEquals(v7, resSet.getBoolean(8)); - assertEquals(v7, resSet.getBoolean("k7")); - - assertEquals(v8, resSet.getString(9)); - assertEquals(v8, resSet.getString("k8")); - - resSet.getBytes(9); - resSet.getObject(6); - resSet.getObject("k8"); - } - if (!resSet.isClosed()) { - resSet.close(); - } - } catch (SQLException e) { - assert false : "insert error " + e.getMessage(); - } - } - - @Test - public void testUnsupport() throws SQLException { - statement.executeQuery("show databases"); - resSet = statement.getResultSet(); - try { - resSet.unwrap(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isWrapperFor(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getAsciiStream(0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getUnicodeStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getBinaryStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getAsciiStream(""); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getUnicodeStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getBinaryStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getWarnings(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.clearWarnings(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getCursorName(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getCharacterStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getCharacterStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isBeforeFirst(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isAfterLast(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isFirst(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.isLast(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.beforeFirst(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.afterLast(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.first(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.last(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.absolute(1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.relative(1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.previous(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.setFetchDirection(0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getFetchDirection(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.setFetchSize(0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getFetchSize(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getConcurrency(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.rowUpdated(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.rowInserted(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.rowDeleted(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNull(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBoolean(0, true); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateByte(0, (byte) 2); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateShort(0, (short) 1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateInt(0, 0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateLong(0, 0l); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateFloat(0, 3.14f); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateDouble(0, 3.1415); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBigDecimal(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateString(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBytes(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateDate(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateTime(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateTimestamp(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateObject(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateObject(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNull(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBoolean("", false); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateByte("", (byte) 1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateShort("", (short) 1); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateInt("", 0); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateLong("", 0l); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateFloat("", 3.14f); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateDouble("", 3.1415); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBigDecimal(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateString(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBytes(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateDate(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateTime(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateTimestamp(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateObject(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateObject(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.insertRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.deleteRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.refreshRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.cancelRowUpdates(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.moveToInsertRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.moveToCurrentRow(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getStatement(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getObject(0, new HashMap<>()); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRef(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getBlob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getClob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getArray(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getObject("", new HashMap<>()); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRef(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getBlob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getClob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getArray(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getDate(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getDate(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getTime(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getTime(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getTimestamp(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getTimestamp(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getURL(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getURL(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRef(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRef(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBlob(0, new SerialBlob("".getBytes("UTF8"))); - } catch (Exception e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBlob("", new SerialBlob("".getBytes("UTF8"))); - } catch (Exception e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateClob("", new SerialClob("".toCharArray())); - } catch (Exception e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateClob(0, new SerialClob("".toCharArray())); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateArray(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateArray(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRowId(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getRowId(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRowId(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateRowId(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getHoldability(); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNString(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNString(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - - try { - resSet.getNClob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getNClob(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getSQLXML(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getSQLXML(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateSQLXML(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateSQLXML(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getNCharacterStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.getNCharacterStream(null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - - try { - resSet.updateNCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateNCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateAsciiStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateBinaryStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - try { - resSet.updateCharacterStream(null, null); - } catch (SQLException e) { - assertTrue(e.getMessage().contains("this operation is NOT supported currently!")); - } - } - - @Test - public void testBatch() throws SQLException { - String[] sqls = new String[]{"insert into test.t0 values (1496732686001,2147483600,1496732687000,3.1415925,3.1415926535897," + - "'涛思数据,强~',12,0,'TDengine is powerful')", "insert into test.t0 values (1496732686002,2147483600,1496732687000,3.1415925,3.1415926535897," + - "'涛思数据,强~',12,1,'TDengine is powerful')"}; - for (String sql : sqls) { - statement.addBatch(sql); - } - int[] res = statement.executeBatch(); - assertEquals(res.length, 2); - statement.clearBatch(); - } - @AfterClass - public static void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } - -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java deleted file mode 100644 index 6c75860e0f0a2647175de6e836f7f1aeb62c539a..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.sql.*; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; - -public class SelectTest extends BaseTest { - Connection connection = null; - Statement statement = null; - String dbName = "test"; - String tName = "t0"; - String host = "localhost"; - - @Before - public void createDatabaseAndTable() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - } - - @Test - public void selectData() throws SQLException { - long ts = 1496732686000l; - - for (int i = 0; i < 50; i++) { - ts++; - int row = statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); - System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); - assertEquals(1, row); - } - - String sql = "select * from " + dbName + "." + tName; - ResultSet resSet = statement.executeQuery(sql); - - int num = 0; - while (resSet.next()) { - num++; - } - resSet.close(); - - assertEquals(num, 50); - } - - @After - public void close() throws Exception { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java deleted file mode 100644 index 6e01fb7c349bbab73fcc6878d85330ac39de2cfc..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.taosdata.jdbc; - -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -import java.sql.*; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; - -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class StableTest extends BaseTest { - static Connection connection = null; - static Statement statement = null; - static String dbName = "test"; - static String stbName = "st"; - static String host = "localhost"; - - @BeforeClass - public static void createDatabase() throws SQLException { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeQuery("use " + dbName); - } - -// @Test - public void createStable() { - String sql = "create table " + stbName + " (ts timestamp, v1 int, v2 int) tags (tg nchar(20)) "; - - try { - statement.executeUpdate(sql); - } catch (SQLException e) { - assert false : "error create stable" + e.getMessage(); - } - } - -// @Test - public void createTable() { - String sql = "create table t1 using " + stbName + " tags (\"beijing\")"; - - try { - statement.executeUpdate(sql); - } catch (SQLException e) { - assert false : "error create table" + e.getMessage(); - } - } - - @Test - public void describeSTable() { - createStable(); - String sql = "describe " + stbName; - int num = 0; - System.out.println("describe stable"); - try { - ResultSet res = statement.executeQuery(sql); - while (res.next()) { - for (int i = 1; i <= res.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s\n", i, res.getString(i)); - } - num++; - } - res.close(); - assertEquals(4, num); - } catch (SQLException e) { - assert false : "error describe stable" + e.getMessage(); - } - } - - @Test - public void describeTable() { - createTable(); - String sql = "describe t1"; - int num = 0; - System.out.println("describe table"); - try { - ResultSet res = statement.executeQuery(sql); - while (res.next()) { - for (int i = 1; i <= res.getMetaData().getColumnCount(); i++) { - System.out.printf("%d: %s\n", i, res.getString(i)); - } - num++; - } - res.close(); - assertEquals(4, num); - } catch (SQLException e) { - assert false : "error describe stable" + e.getMessage(); - } - } - - // @Test - public void validCreateSql() { - String sql = "create table t2 using " + stbName + " tags (\"beijing\")"; - boolean valid = ((TSDBConnection) connection).getConnection().validateCreateTableSql(sql); - assertEquals(true, valid); - } - - @AfterClass - public static void close() throws Exception { - if (!statement.isClosed()) { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - - } - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index db7b8c8cb14cbc0d1ae5a5a3a3bf8df590c4bc2a..73ceafa7299b256d7e83064b53bd638835a4b075 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -1,40 +1,51 @@ package com.taosdata.jdbc; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import java.sql.*; import java.util.Properties; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class StatementTest extends BaseTest { +public class StatementTest { static Connection connection = null; static Statement statement = null; static String dbName = "test"; static String tName = "t0"; static String host = "localhost"; - static ResultSet resSet = null; @BeforeClass public static void createConnection() throws SQLException { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/?user=root&password=taosdata", properties); + statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + } catch (ClassNotFoundException e) { return; } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - - statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); + } + @Test + public void testCase() { + try { + ResultSet rs = statement.executeQuery("show databases"); + ResultSetMetaData metaData = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + System.out.print(metaData.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } } @Test @@ -44,130 +55,54 @@ public class StatementTest extends BaseTest { statement.executeUpdate("create database if not exists " + dbName); statement.executeUpdate("create table if not exists " + dbName + "." + tName + "(ts timestamp, k1 int)"); statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 1)"); - statement.executeQuery("select * from " + dbName + "." + tName); + statement.execute("select * from " + dbName + "." + tName); ResultSet resultSet = statement.getResultSet(); - assertTrue(null != resultSet); + Assert.assertNotNull(resultSet); boolean isClosed = statement.isClosed(); - assertEquals(false, isClosed); + Assert.assertEquals(false, isClosed); } - @Test - public void testUnsupport() { -// if(null == resSet) { -// return; -// } - TSDBStatement tsdbStatement = (TSDBStatement) statement; - try { - tsdbStatement.unwrap(null); - } catch (SQLException e) { - } - try { - tsdbStatement.isWrapperFor(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMaxFieldSize(); - } catch (SQLException e) { - } - try { - tsdbStatement.setMaxFieldSize(0); - } catch (SQLException e) { - } - try { - tsdbStatement.setEscapeProcessing(true); - } catch (SQLException e) { - } - try { - tsdbStatement.cancel(); - } catch (SQLException e) { - } - try { - tsdbStatement.getWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.clearWarnings(); - } catch (SQLException e) { - } - try { - tsdbStatement.setCursorName(null); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.setFetchDirection(0); - } catch (SQLException e) { - } - try { - tsdbStatement.getFetchDirection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetConcurrency(); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetType(); - } catch (SQLException e) { - } - try { - tsdbStatement.getConnection(); - } catch (SQLException e) { - } - try { - tsdbStatement.getMoreResults(); - } catch (SQLException e) { - } - try { - tsdbStatement.getGeneratedKeys(); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, 0); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new int[]{0}); - } catch (SQLException e) { - } - try { - tsdbStatement.executeUpdate(null, new String[]{"str1", "str2"}); - } catch (SQLException e) { - } - try { - tsdbStatement.getResultSetHoldability(); - } catch (SQLException e) { - } - try { - tsdbStatement.setPoolable(true); - } catch (SQLException e) { - } - try { - tsdbStatement.isPoolable(); - } catch (SQLException e) { - } - try { - tsdbStatement.closeOnCompletion(); - } catch (SQLException e) { - } - try { - tsdbStatement.isCloseOnCompletion(); - } catch (SQLException e) { - } + @Test(expected = SQLException.class) + public void testUnsupport() throws SQLException { + Assert.assertNotNull(statement.unwrap(TSDBStatement.class)); + Assert.assertTrue(statement.isWrapperFor(TSDBStatement.class)); + + statement.getMaxFieldSize(); + statement.setMaxFieldSize(0); + statement.setEscapeProcessing(true); + statement.cancel(); + statement.getWarnings(); + statement.clearWarnings(); + statement.setCursorName(null); + statement.getMoreResults(); + statement.setFetchDirection(0); + statement.getFetchDirection(); + statement.getResultSetConcurrency(); + statement.getResultSetType(); + statement.getConnection(); + statement.getMoreResults(); + statement.getGeneratedKeys(); + statement.executeUpdate(null, 0); + statement.executeUpdate(null, new int[]{0}); + statement.executeUpdate(null, new String[]{"str1", "str2"}); + statement.getResultSetHoldability(); + statement.setPoolable(true); + statement.isPoolable(); + statement.closeOnCompletion(); + statement.isCloseOnCompletion(); } @AfterClass - public static void close() throws Exception { - if (!statement.isClosed()) { - statement.executeUpdate("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); - + public static void close() { + try { + statement.execute("drop database if exists " + dbName); + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); } } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 0a71c77d1d9dc718cc029c6a0b11ed63cd3034c1..11c3de305280a48b690ccdacd2df9751f4b718a9 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -9,80 +9,83 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; +import java.util.concurrent.TimeUnit; -public class SubscribeTest extends BaseTest { - Connection connection = null; - Statement statement = null; +public class SubscribeTest { + Connection connection; + Statement statement; String dbName = "test"; String tName = "t0"; - String host = "localhost"; + String host = "127.0.0.1"; String topic = "test"; @Before - public void createDatabase() throws SQLException { + public void createDatabase() { try { Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - return; - } - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + statement = connection.createStatement(); + statement.execute("drop database if exists " + dbName); + statement.execute("create database if not exists " + dbName); + statement.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); + long ts = System.currentTimeMillis(); + for (int i = 0; i < 2; i++) { + ts += i; + String sql = "insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"; + statement.executeUpdate(sql); + } - statement = connection.createStatement(); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - long ts = System.currentTimeMillis(); - for (int i = 0; i < 2; i++) { - ts += i; - String sql = "insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"; - statement.executeUpdate(sql); + } catch (ClassNotFoundException | SQLException e) { + return; } } @Test - public void subscribe() throws Exception { - TSDBSubscribe subscribe = null; + public void subscribe() { try { - String rawSql = "select * from " + dbName + "." + tName + ";"; System.out.println(rawSql); - subscribe = ((TSDBConnection) connection).subscribe(topic, rawSql, false); - - int a = 0; - while (true) { - Thread.sleep(900); - TSDBResultSet resSet = subscribe.consume(); +// TSDBSubscribe subscribe = ((TSDBConnection) connection).subscribe(topic, rawSql, false); - while (resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf(i + ": " + resSet.getString(i) + "\t"); - } - System.out.println("\n======" + a + "=========="); - } - resSet.close(); - a++; - if (a >= 2) { - break; - } - } +// int a = 0; +// while (true) { +// TimeUnit.MILLISECONDS.sleep(1000); +// TSDBResultSet resSet = subscribe.consume(); +// while (resSet.next()) { +// for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { +// System.out.printf(i + ": " + resSet.getString(i) + "\t"); +// } +// System.out.println("\n======" + a + "=========="); +// } +// a++; +// if (a >= 2) { +// break; +// } +// resSet.close(); +// } +// +// subscribe.close(true); } catch (Exception e) { e.printStackTrace(); - } finally { - if (null != subscribe) { - subscribe.close(true); - } } } @After - public void close() throws Exception { - statement.executeQuery("drop database " + dbName); - statement.close(); - connection.close(); - Thread.sleep(10); + public void close() { + try { + statement.execute("drop database " + dbName); + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java new file mode 100644 index 0000000000000000000000000000000000000000..ca7251bb0eac306d6a21872699918e18b2447e6e --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBConnectionTest.java @@ -0,0 +1,423 @@ +package com.taosdata.jdbc; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import javax.management.OperationsException; +import java.sql.*; +import java.util.Properties; + +public class TSDBConnectionTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + + @Test + public void getConnection() { + // already test in beforeClass method + } + + @Test + public void createStatement() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void subscribe() { + try { + TSDBConnection unwrap = conn.unwrap(TSDBConnection.class); + TSDBSubscribe subscribe = unwrap.subscribe("topic1", "select * from log.log", false); + TSDBResultSet rs = subscribe.consume(); + ResultSetMetaData metaData = rs.getMetaData(); + for (int count = 0; count < 10 && rs.next(); count++) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String value = rs.getString(i); + System.out.print(metaData.getColumnLabel(i) + ":" + value + "\t"); + } + System.out.println(); + } + Assert.assertNotNull(rs); + subscribe.close(false); + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @Test + public void prepareStatement() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()"); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void prepareCall() throws SQLException { + conn.prepareCall("select server_status()"); + } + + @Test + public void nativeSQL() throws SQLException { + String nativeSQL = conn.nativeSQL("select * from log.log"); + Assert.assertEquals("select * from log.log", nativeSQL); + } + + @Test + public void setAutoCommit() throws SQLException { + conn.setAutoCommit(true); + conn.setAutoCommit(false); + } + + @Test + public void getAutoCommit() throws SQLException { + Assert.assertTrue(conn.getAutoCommit()); + } + + @Test + public void commit() throws SQLException { + conn.commit(); + } + + @Test + public void rollback() throws SQLException { + conn.rollback(); + } + + @Test + public void close() { + // connection will close in afterClass method + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(conn.isClosed()); + } + + @Test + public void getMetaData() throws SQLException { + DatabaseMetaData meta = conn.getMetaData(); + Assert.assertNotNull(meta); + Assert.assertEquals("com.taosdata.jdbc.TSDBDriver", meta.getDriverName()); + } + + @Test + public void setReadOnly() throws SQLException { + conn.setReadOnly(true); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertTrue(conn.isReadOnly()); + } + + @Test + public void setCatalog() throws SQLException { + conn.setCatalog("test"); + Assert.assertEquals("test", conn.getCatalog()); + } + + @Test + public void getCatalog() throws SQLException { + conn.setCatalog("log"); + Assert.assertEquals("log", conn.getCatalog()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTransactionIsolation() throws SQLException { + conn.setTransactionIsolation(Connection.TRANSACTION_NONE); + Assert.assertEquals(Connection.TRANSACTION_NONE, conn.getTransactionIsolation()); + conn.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); + } + + @Test + public void getTransactionIsolation() throws SQLException { + Assert.assertEquals(Connection.TRANSACTION_NONE, conn.getTransactionIsolation()); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(conn.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + conn.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testCreateStatement() throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()", + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.prepareStatement("select server_status", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareCall() throws SQLException { + conn.prepareCall("", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getTypeMap() throws SQLException { + conn.getTypeMap(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTypeMap() throws SQLException { + conn.setTypeMap(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setHoldability() throws SQLException { + conn.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, conn.getHoldability()); + conn.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, conn.getHoldability()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setSavepoint() throws SQLException { + conn.setSavepoint(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testSetSavepoint() throws SQLException { + conn.setSavepoint(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testRollback() throws SQLException { + conn.rollback(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void releaseSavepoint() throws SQLException { + conn.releaseSavepoint(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testCreateStatement1() throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement1() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()", + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.prepareStatement("select server_status", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareCall1() throws SQLException { + conn.prepareCall("", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement2() throws SQLException { + Assert.assertNotNull("", Statement.NO_GENERATED_KEYS); + conn.prepareStatement("", Statement.RETURN_GENERATED_KEYS); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement3() throws SQLException { + conn.prepareStatement("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement4() throws SQLException { + conn.prepareStatement("", new String[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createClob() throws SQLException { + conn.createClob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createBlob() throws SQLException { + conn.createBlob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createNClob() throws SQLException { + conn.createNClob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createSQLXML() throws SQLException { + conn.createSQLXML(); + } + + @Test(expected = SQLException.class) + public void isValid() throws SQLException { + Assert.assertTrue(conn.isValid(10)); + Assert.assertTrue(conn.isValid(0)); + conn.isValid(-1); + } + + @Test + public void setClientInfo() throws SQLClientInfoException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTC-8"); + } + + @Test + public void testSetClientInfo() throws SQLClientInfoException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn.setClientInfo(properties); + } + + @Test + public void getClientInfo() throws SQLException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + Properties info = conn.getClientInfo(); + String charset = info.getProperty(TSDBDriver.PROPERTY_KEY_CHARSET); + Assert.assertEquals("UTF-8", charset); + String locale = info.getProperty(TSDBDriver.PROPERTY_KEY_LOCALE); + Assert.assertEquals("en_US.UTF-8", locale); + String timezone = info.getProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE); + Assert.assertEquals("UTC-8", timezone); + } + + @Test + public void testGetClientInfo() throws SQLException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String charset = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET); + Assert.assertEquals("UTF-8", charset); + String locale = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE); + Assert.assertEquals("en_US.UTF-8", locale); + String timezone = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE); + Assert.assertEquals("UTC-8", timezone); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createArrayOf() throws SQLException { + conn.createArrayOf("", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createStruct() throws SQLException { + conn.createStruct("", null); + } + + @Test + public void setSchema() throws SQLException { + conn.setSchema("test"); + } + + @Test + public void getSchema() throws SQLException { + Assert.assertNull(conn.getSchema()); + } + + @Test + public void abort() throws SQLException { + conn.abort(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNetworkTimeout() throws SQLException { + conn.setNetworkTimeout(null, 1000); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNetworkTimeout() throws SQLException { + conn.getNetworkTimeout(); + } + + @Test + public void unwrap() { + try { + TSDBConnection tsdbConnection = conn.unwrap(TSDBConnection.class); + Assert.assertNotNull(tsdbConnection); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(conn.isWrapperFor(TSDBConnection.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/log?user=root&password=taosdata", properties); + // create test database for test cases + try (Statement stmt = conn.createStatement()) { + stmt.execute("create database if not exists test"); + } + + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a7657fa3e581c96cf85555b16fca5f8b4c7f8ebb --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDatabaseMetaDataTest.java @@ -0,0 +1,978 @@ +package com.taosdata.jdbc; + +import org.junit.*; + +import java.sql.*; +import java.util.Properties; + +public class TSDBDatabaseMetaDataTest { + private static final String host = "127.0.0.1"; + private static Connection connection; + private static TSDBDatabaseMetaData metaData; + + @Test + public void unwrap() throws SQLException { + TSDBDatabaseMetaData unwrap = metaData.unwrap(TSDBDatabaseMetaData.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(metaData.isWrapperFor(TSDBDatabaseMetaData.class)); + } + + @Test + public void allProceduresAreCallable() throws SQLException { + Assert.assertFalse(metaData.allProceduresAreCallable()); + } + + @Test + public void allTablesAreSelectable() throws SQLException { + Assert.assertFalse(metaData.allTablesAreSelectable()); + } + + @Test + public void getURL() throws SQLException { + Assert.assertEquals("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", metaData.getURL()); + } + + @Test + public void getUserName() throws SQLException { + Assert.assertEquals("root", metaData.getUserName()); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertFalse(metaData.isReadOnly()); + } + + @Test + public void nullsAreSortedHigh() throws SQLException { + Assert.assertFalse(metaData.nullsAreSortedHigh()); + } + + @Test + public void nullsAreSortedLow() throws SQLException { + Assert.assertTrue(metaData.nullsAreSortedLow()); + } + + @Test + public void nullsAreSortedAtStart() throws SQLException { + Assert.assertTrue(metaData.nullsAreSortedAtStart()); + } + + @Test + public void nullsAreSortedAtEnd() throws SQLException { + Assert.assertFalse(metaData.nullsAreSortedAtEnd()); + } + + @Test + public void getDatabaseProductName() throws SQLException { + Assert.assertEquals("TDengine", metaData.getDatabaseProductName()); + } + + @Test + public void getDatabaseProductVersion() throws SQLException { + Assert.assertEquals("2.0.x.x", metaData.getDatabaseProductVersion()); + } + + @Test + public void getDriverName() throws SQLException { + Assert.assertEquals("com.taosdata.jdbc.TSDBDriver", metaData.getDriverName()); + } + + @Test + public void getDriverVersion() throws SQLException { + Assert.assertEquals("2.0.x", metaData.getDriverVersion()); + } + + @Test + public void getDriverMajorVersion() { + Assert.assertEquals(2, metaData.getDriverMajorVersion()); + } + + @Test + public void getDriverMinorVersion() { + Assert.assertEquals(0, metaData.getDriverMinorVersion()); + } + + @Test + public void usesLocalFiles() throws SQLException { + Assert.assertFalse(metaData.usesLocalFiles()); + } + + @Test + public void usesLocalFilePerTable() throws SQLException { + Assert.assertFalse(metaData.usesLocalFilePerTable()); + } + + @Test + public void supportsMixedCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.supportsMixedCaseIdentifiers()); + } + + @Test + public void storesUpperCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesUpperCaseIdentifiers()); + } + + @Test + public void storesLowerCaseIdentifiers() throws SQLException { + Assert.assertTrue(metaData.storesLowerCaseIdentifiers()); + } + + @Test + public void storesMixedCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesMixedCaseIdentifiers()); + } + + @Test + public void supportsMixedCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.supportsMixedCaseQuotedIdentifiers()); + } + + @Test + public void storesUpperCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesUpperCaseQuotedIdentifiers()); + } + + @Test + public void storesLowerCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesLowerCaseQuotedIdentifiers()); + } + + @Test + public void storesMixedCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesMixedCaseQuotedIdentifiers()); + } + + @Test + public void getIdentifierQuoteString() throws SQLException { + Assert.assertEquals(" ", metaData.getIdentifierQuoteString()); + } + + @Test + public void getSQLKeywords() throws SQLException { + Assert.assertEquals(null, metaData.getSQLKeywords()); + } + + @Test + public void getNumericFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getNumericFunctions()); + } + + @Test + public void getStringFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getStringFunctions()); + } + + @Test + public void getSystemFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getSystemFunctions()); + } + + @Test + public void getTimeDateFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getTimeDateFunctions()); + } + + @Test + public void getSearchStringEscape() throws SQLException { + Assert.assertEquals(null, metaData.getSearchStringEscape()); + } + + @Test + public void getExtraNameCharacters() throws SQLException { + Assert.assertEquals(null, metaData.getExtraNameCharacters()); + } + + @Test + public void supportsAlterTableWithAddColumn() throws SQLException { + Assert.assertTrue(metaData.supportsAlterTableWithAddColumn()); + } + + @Test + public void supportsAlterTableWithDropColumn() throws SQLException { + Assert.assertTrue(metaData.supportsAlterTableWithDropColumn()); + } + + @Test + public void supportsColumnAliasing() throws SQLException { + Assert.assertTrue(metaData.supportsColumnAliasing()); + } + + @Test + public void nullPlusNonNullIsNull() throws SQLException { + Assert.assertFalse(metaData.nullPlusNonNullIsNull()); + } + + @Test + public void supportsConvert() throws SQLException { + Assert.assertFalse(metaData.supportsConvert()); + } + + @Test + public void testSupportsConvert() throws SQLException { + Assert.assertFalse(metaData.supportsConvert(1, 1)); + } + + @Test + public void supportsTableCorrelationNames() throws SQLException { + Assert.assertFalse(metaData.supportsTableCorrelationNames()); + } + + @Test + public void supportsDifferentTableCorrelationNames() throws SQLException { + Assert.assertFalse(metaData.supportsDifferentTableCorrelationNames()); + } + + @Test + public void supportsExpressionsInOrderBy() throws SQLException { + Assert.assertFalse(metaData.supportsExpressionsInOrderBy()); + } + + @Test + public void supportsOrderByUnrelated() throws SQLException { + Assert.assertFalse(metaData.supportsOrderByUnrelated()); + } + + @Test + public void supportsGroupBy() throws SQLException { + Assert.assertTrue(metaData.supportsGroupBy()); + } + + @Test + public void supportsGroupByUnrelated() throws SQLException { + Assert.assertFalse(metaData.supportsGroupByUnrelated()); + } + + @Test + public void supportsGroupByBeyondSelect() throws SQLException { + Assert.assertFalse(metaData.supportsGroupByBeyondSelect()); + } + + @Test + public void supportsLikeEscapeClause() throws SQLException { + Assert.assertFalse(metaData.supportsLikeEscapeClause()); + } + + @Test + public void supportsMultipleResultSets() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleResultSets()); + } + + @Test + public void supportsMultipleTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleTransactions()); + } + + @Test + public void supportsNonNullableColumns() throws SQLException { + Assert.assertFalse(metaData.supportsNonNullableColumns()); + } + + @Test + public void supportsMinimumSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsMinimumSQLGrammar()); + } + + @Test + public void supportsCoreSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsCoreSQLGrammar()); + } + + @Test + public void supportsExtendedSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsExtendedSQLGrammar()); + } + + @Test + public void supportsANSI92EntryLevelSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92EntryLevelSQL()); + } + + @Test + public void supportsANSI92IntermediateSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92IntermediateSQL()); + } + + @Test + public void supportsANSI92FullSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92FullSQL()); + } + + @Test + public void supportsIntegrityEnhancementFacility() throws SQLException { + Assert.assertFalse(metaData.supportsIntegrityEnhancementFacility()); + } + + @Test + public void supportsOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsOuterJoins()); + } + + @Test + public void supportsFullOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsFullOuterJoins()); + } + + @Test + public void supportsLimitedOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsLimitedOuterJoins()); + } + + @Test + public void getSchemaTerm() throws SQLException { + Assert.assertNull(metaData.getSchemaTerm()); + } + + @Test + public void getProcedureTerm() throws SQLException { + Assert.assertNull(metaData.getProcedureTerm()); + } + + @Test + public void getCatalogTerm() throws SQLException { + Assert.assertEquals("database", metaData.getCatalogTerm()); + } + + @Test + public void isCatalogAtStart() throws SQLException { + Assert.assertTrue(metaData.isCatalogAtStart()); + } + + @Test + public void getCatalogSeparator() throws SQLException { + Assert.assertEquals(".", metaData.getCatalogSeparator()); + } + + @Test + public void supportsSchemasInDataManipulation() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInDataManipulation()); + } + + @Test + public void supportsSchemasInProcedureCalls() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInProcedureCalls()); + } + + @Test + public void supportsSchemasInTableDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInTableDefinitions()); + } + + @Test + public void supportsSchemasInIndexDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInIndexDefinitions()); + } + + @Test + public void supportsSchemasInPrivilegeDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInPrivilegeDefinitions()); + } + + @Test + public void supportsCatalogsInDataManipulation() throws SQLException { + Assert.assertTrue(metaData.supportsCatalogsInDataManipulation()); + } + + @Test + public void supportsCatalogsInProcedureCalls() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInProcedureCalls()); + } + + @Test + public void supportsCatalogsInTableDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInTableDefinitions()); + } + + @Test + public void supportsCatalogsInIndexDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInIndexDefinitions()); + } + + @Test + public void supportsCatalogsInPrivilegeDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInPrivilegeDefinitions()); + } + + @Test + public void supportsPositionedDelete() throws SQLException { + Assert.assertFalse(metaData.supportsPositionedDelete()); + } + + @Test + public void supportsPositionedUpdate() throws SQLException { + Assert.assertFalse(metaData.supportsPositionedUpdate()); + } + + @Test + public void supportsSelectForUpdate() throws SQLException { + Assert.assertFalse(metaData.supportsSelectForUpdate()); + } + + @Test + public void supportsStoredProcedures() throws SQLException { + Assert.assertFalse(metaData.supportsStoredProcedures()); + } + + @Test + public void supportsSubqueriesInComparisons() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInComparisons()); + } + + @Test + public void supportsSubqueriesInExists() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInExists()); + } + + @Test + public void supportsSubqueriesInIns() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInIns()); + } + + @Test + public void supportsSubqueriesInQuantifieds() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInQuantifieds()); + } + + @Test + public void supportsCorrelatedSubqueries() throws SQLException { + Assert.assertFalse(metaData.supportsCorrelatedSubqueries()); + } + + @Test + public void supportsUnion() throws SQLException { + Assert.assertFalse(metaData.supportsUnion()); + } + + @Test + public void supportsUnionAll() throws SQLException { + Assert.assertFalse(metaData.supportsUnionAll()); + } + + @Test + public void supportsOpenCursorsAcrossCommit() throws SQLException { + Assert.assertFalse(metaData.supportsOpenCursorsAcrossCommit()); + } + + @Test + public void supportsOpenCursorsAcrossRollback() throws SQLException { + Assert.assertFalse(metaData.supportsOpenCursorsAcrossRollback()); + } + + @Test + public void supportsOpenStatementsAcrossCommit() throws SQLException { + Assert.assertFalse(metaData.supportsOpenStatementsAcrossCommit()); + } + + @Test + public void supportsOpenStatementsAcrossRollback() throws SQLException { + Assert.assertFalse(metaData.supportsOpenStatementsAcrossRollback()); + } + + @Test + public void getMaxBinaryLiteralLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxBinaryLiteralLength()); + } + + @Test + public void getMaxCharLiteralLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCharLiteralLength()); + } + + @Test + public void getMaxColumnNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnNameLength()); + } + + @Test + public void getMaxColumnsInGroupBy() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInGroupBy()); + } + + @Test + public void getMaxColumnsInIndex() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInIndex()); + } + + @Test + public void getMaxColumnsInOrderBy() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInOrderBy()); + } + + @Test + public void getMaxColumnsInSelect() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInSelect()); + } + + @Test + public void getMaxColumnsInTable() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInTable()); + } + + @Test + public void getMaxConnections() throws SQLException { + Assert.assertEquals(0, metaData.getMaxConnections()); + } + + @Test + public void getMaxCursorNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCursorNameLength()); + } + + @Test + public void getMaxIndexLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxIndexLength()); + } + + @Test + public void getMaxSchemaNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxSchemaNameLength()); + } + + @Test + public void getMaxProcedureNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxProcedureNameLength()); + } + + @Test + public void getMaxCatalogNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCatalogNameLength()); + } + + @Test + public void getMaxRowSize() throws SQLException { + Assert.assertEquals(0, metaData.getMaxRowSize()); + } + + @Test + public void doesMaxRowSizeIncludeBlobs() throws SQLException { + Assert.assertFalse(metaData.doesMaxRowSizeIncludeBlobs()); + } + + @Test + public void getMaxStatementLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxStatementLength()); + } + + @Test + public void getMaxStatements() throws SQLException { + Assert.assertEquals(0, metaData.getMaxStatements()); + } + + @Test + public void getMaxTableNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxTableNameLength()); + } + + @Test + public void getMaxTablesInSelect() throws SQLException { + Assert.assertEquals(0, metaData.getMaxTablesInSelect()); + } + + @Test + public void getMaxUserNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxUserNameLength()); + } + + @Test + public void getDefaultTransactionIsolation() throws SQLException { + Assert.assertEquals(Connection.TRANSACTION_NONE, metaData.getDefaultTransactionIsolation()); + } + + @Test + public void supportsTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsTransactions()); + } + + @Test + public void supportsTransactionIsolationLevel() throws SQLException { + Assert.assertTrue(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_NONE)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_COMMITTED)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_UNCOMMITTED)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_REPEATABLE_READ)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_SERIALIZABLE)); + } + + @Test + public void supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsDataDefinitionAndDataManipulationTransactions()); + } + + @Test + public void supportsDataManipulationTransactionsOnly() throws SQLException { + Assert.assertFalse(metaData.supportsDataManipulationTransactionsOnly()); + } + + @Test + public void dataDefinitionCausesTransactionCommit() throws SQLException { + Assert.assertFalse(metaData.dataDefinitionCausesTransactionCommit()); + } + + @Test + public void dataDefinitionIgnoredInTransactions() throws SQLException { + Assert.assertFalse(metaData.dataDefinitionIgnoredInTransactions()); + } + + @Test + public void getProcedures() throws SQLException { + Assert.assertNull(metaData.getProcedures("*", "*", "*")); + } + + @Test + public void getProcedureColumns() throws SQLException { + Assert.assertNull(metaData.getProcedureColumns("*", "*", "*", "*")); + } + + @Test + public void getTables() throws SQLException { + System.out.println("****************************************************"); + ResultSet tables = metaData.getTables("log", "", null, null); + ResultSetMetaData metaData = tables.getMetaData(); + while (tables.next()) { + System.out.print(metaData.getColumnLabel(1) + ":" + tables.getString(1) + "\t"); + System.out.print(metaData.getColumnLabel(3) + ":" + tables.getString(3) + "\t"); + System.out.print(metaData.getColumnLabel(4) + ":" + tables.getString(4) + "\t"); + System.out.print(metaData.getColumnLabel(5) + ":" + tables.getString(5) + "\n"); + } + System.out.println(); + Assert.assertNotNull(tables); + } + + @Test + public void getSchemas() throws SQLException { + Assert.assertNotNull(metaData.getSchemas()); + } + + @Test + public void getCatalogs() throws SQLException { + System.out.println("****************************************************"); + + ResultSet catalogs = metaData.getCatalogs(); + ResultSetMetaData meta = catalogs.getMetaData(); + while (catalogs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + catalogs.getString(i)); + } + System.out.println(); + } + } + + @Test + public void getTableTypes() throws SQLException { + System.out.println("****************************************************"); + + ResultSet tableTypes = metaData.getTableTypes(); + while (tableTypes.next()) { + System.out.println(tableTypes.getString("TABLE_TYPE")); + } + Assert.assertNotNull(metaData.getTableTypes()); + } + + @Test + public void getColumns() throws SQLException { + System.out.println("****************************************************"); + + ResultSet columns = metaData.getColumns("log", "", "dn", ""); + ResultSetMetaData meta = columns.getMetaData(); + while (columns.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + columns.getString(1) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + columns.getString(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + columns.getString(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + columns.getString(5) + "\t"); + System.out.print(meta.getColumnLabel(6) + ": " + columns.getString(6) + "\t"); + System.out.print(meta.getColumnLabel(7) + ": " + columns.getString(7) + "\t"); + System.out.print(meta.getColumnLabel(9) + ": " + columns.getString(9) + "\t"); + System.out.print(meta.getColumnLabel(10) + ": " + columns.getString(10) + "\t"); + System.out.print(meta.getColumnLabel(11) + ": " + columns.getString(11) + "\n"); + System.out.print(meta.getColumnLabel(12) + ": " + columns.getString(12) + "\n"); + } + } + + @Test + public void getColumnPrivileges() throws SQLException { + Assert.assertNotNull(metaData.getColumnPrivileges("", "", "", "")); + } + + @Test + public void getTablePrivileges() throws SQLException { + Assert.assertNotNull(metaData.getTablePrivileges("", "", "")); + } + + @Test + public void getBestRowIdentifier() throws SQLException { + Assert.assertNotNull(metaData.getBestRowIdentifier("", "", "", 0, false)); + } + + @Test + public void getVersionColumns() throws SQLException { + Assert.assertNotNull(metaData.getVersionColumns("", "", "")); + } + + @Test + public void getPrimaryKeys() throws SQLException { + System.out.println("****************************************************"); + + ResultSet rs = metaData.getPrimaryKeys("log", "", "dn1"); + while (rs.next()) { + System.out.println("TABLE_NAME: " + rs.getString("TABLE_NAME")); + System.out.println("COLUMN_NAME: " + rs.getString("COLUMN_NAME")); + System.out.println("KEY_SEQ: " + rs.getString("KEY_SEQ")); + System.out.println("PK_NAME: " + rs.getString("PK_NAME")); + } + + Assert.assertNotNull(rs); + } + + @Test + public void getImportedKeys() throws SQLException { + Assert.assertNotNull(metaData.getImportedKeys("", "", "")); + } + + @Test + public void getExportedKeys() throws SQLException { + Assert.assertNotNull(metaData.getExportedKeys("", "", "")); + } + + @Test + public void getCrossReference() throws SQLException { + Assert.assertNotNull(metaData.getCrossReference("", "", "", "", "", "")); + } + + @Test + public void getTypeInfo() throws SQLException { + Assert.assertNotNull(metaData.getTypeInfo()); + } + + @Test + public void getIndexInfo() throws SQLException { + Assert.assertNotNull(metaData.getIndexInfo("", "", "", false, false)); + } + + @Test + public void supportsResultSetType() throws SQLException { + Assert.assertFalse(metaData.supportsResultSetType(0)); + } + + @Test + public void supportsResultSetConcurrency() throws SQLException { + Assert.assertFalse(metaData.supportsResultSetConcurrency(0, 0)); + } + + @Test + public void ownUpdatesAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownUpdatesAreVisible(0)); + } + + @Test + public void ownDeletesAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownDeletesAreVisible(0)); + } + + @Test + public void ownInsertsAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownInsertsAreVisible(0)); + } + + @Test + public void othersUpdatesAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersUpdatesAreVisible(0)); + } + + @Test + public void othersDeletesAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersDeletesAreVisible(0)); + } + + @Test + public void othersInsertsAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersInsertsAreVisible(0)); + } + + @Test + public void updatesAreDetected() throws SQLException { + Assert.assertFalse(metaData.updatesAreDetected(0)); + } + + @Test + public void deletesAreDetected() throws SQLException { + Assert.assertFalse(metaData.deletesAreDetected(0)); + } + + @Test + public void insertsAreDetected() throws SQLException { + Assert.assertFalse(metaData.insertsAreDetected(0)); + } + + @Test + public void supportsBatchUpdates() throws SQLException { + Assert.assertFalse(metaData.supportsBatchUpdates()); + } + + @Test + public void getUDTs() throws SQLException { + Assert.assertNotNull(metaData.getUDTs("", "", "", null)); + } + + @Test + public void getConnection() throws SQLException { + Assert.assertNotNull(metaData.getConnection()); + } + + @Test + public void supportsSavepoints() throws SQLException { + Assert.assertFalse(metaData.supportsSavepoints()); + } + + @Test + public void supportsNamedParameters() throws SQLException { + Assert.assertFalse(metaData.supportsNamedParameters()); + } + + @Test + public void supportsMultipleOpenResults() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleOpenResults()); + } + + @Test + public void supportsGetGeneratedKeys() throws SQLException { + Assert.assertFalse(metaData.supportsGetGeneratedKeys()); + } + + @Test + public void getSuperTypes() throws SQLException { + Assert.assertNotNull(metaData.getSuperTypes("", "", "")); + } + + @Test + public void getSuperTables() throws SQLException { + System.out.println("****************************************************"); + + ResultSet rs = metaData.getSuperTables("log", "", "dn1"); + while (rs.next()) { + System.out.println("TABLE_NAME: " + rs.getString("TABLE_NAME")); + System.out.println("SUPERTABLE_NAME: " + rs.getString("SUPERTABLE_NAME")); + } + Assert.assertNotNull(rs); + } + + @Test + public void getAttributes() throws SQLException { + Assert.assertNotNull(metaData.getAttributes("", "", "", "")); + } + + @Test + public void supportsResultSetHoldability() throws SQLException { + Assert.assertTrue(metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT)); + Assert.assertFalse(metaData.supportsResultSetHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT)); + } + + @Test + public void getResultSetHoldability() throws SQLException { + Assert.assertEquals(1, metaData.getResultSetHoldability()); + } + + @Test + public void getDatabaseMajorVersion() throws SQLException { + Assert.assertEquals(2, metaData.getDatabaseMajorVersion()); + } + + @Test + public void getDatabaseMinorVersion() throws SQLException { + Assert.assertEquals(0, metaData.getDatabaseMinorVersion()); + } + + @Test + public void getJDBCMajorVersion() throws SQLException { + Assert.assertEquals(2, metaData.getJDBCMajorVersion()); + } + + @Test + public void getJDBCMinorVersion() throws SQLException { + Assert.assertEquals(0, metaData.getJDBCMinorVersion()); + } + + @Test + public void getSQLStateType() throws SQLException { + Assert.assertEquals(0, metaData.getSQLStateType()); + } + + @Test + public void locatorsUpdateCopy() throws SQLException { + Assert.assertFalse(metaData.locatorsUpdateCopy()); + } + + @Test + public void supportsStatementPooling() throws SQLException { + Assert.assertFalse(metaData.supportsStatementPooling()); + } + + @Test + public void getRowIdLifetime() throws SQLException { + Assert.assertNull(metaData.getRowIdLifetime()); + } + + @Test + public void supportsStoredFunctionsUsingCallSyntax() throws SQLException { + Assert.assertFalse(metaData.supportsStoredFunctionsUsingCallSyntax()); + } + + @Test + public void autoCommitFailureClosesAllResultSets() throws SQLException { + Assert.assertFalse(metaData.autoCommitFailureClosesAllResultSets()); + } + + @Test + public void getClientInfoProperties() throws SQLException { + Assert.assertNotNull(metaData.getClientInfoProperties()); + } + + @Test + public void getFunctions() throws SQLException { + Assert.assertNotNull(metaData.getFunctions("", "", "")); + } + + @Test + public void getFunctionColumns() throws SQLException { + Assert.assertNotNull(metaData.getFunctionColumns("", "", "", "")); + } + + @Test + public void getPseudoColumns() throws SQLException { + Assert.assertNotNull(metaData.getPseudoColumns("", "", "", "")); + } + + @Test + public void generatedKeyAlwaysReturned() throws SQLException { + Assert.assertFalse(metaData.generatedKeyAlwaysReturned()); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", properties); + metaData = connection.getMetaData().unwrap(TSDBDatabaseMetaData.class); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 8adcdefb2974d5817793297091eee1c2f40a52b9..671ecd723d6fea8d6b9b8ccf94cba06689ce26b8 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -3,9 +3,6 @@ package com.taosdata.jdbc; import org.junit.BeforeClass; import org.junit.Test; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; import java.sql.*; import java.util.Properties; @@ -27,54 +24,14 @@ public class TSDBDriverTest { "jdbc:TAOS://:/test", "jdbc:TAOS://localhost:0/?user=root&password=taosdata" }; - private static boolean islibLoaded = false; - private static boolean isTaosdActived; - private Connection conn; - @BeforeClass - public static void before() { - String osName = System.getProperty("os.name").toLowerCase(); - if (!osName.equals("linux")) - return; - // try to load taos lib - try { - System.loadLibrary("taos"); - islibLoaded = true; - } catch (UnsatisfiedLinkError error) { - System.out.println("load tdengine lib failed."); - error.printStackTrace(); - } - // check taosd is activated - try { - String[] cmd = {"/bin/bash", "-c", "ps -ef | grep taosd | grep -v \"grep\""}; - Process exec = Runtime.getRuntime().exec(cmd); - BufferedReader reader = new BufferedReader(new InputStreamReader(exec.getInputStream())); - int lineCnt = 0; - while (reader.readLine() != null) { - lineCnt++; - } - if (lineCnt > 0) - isTaosdActived = true; - } catch (IOException e) { - e.printStackTrace(); - } - - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } - @Test - public void testConnectWithJdbcURL() { + public void connectWithJdbcURL() { final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(url); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(url); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); @@ -82,17 +39,15 @@ public class TSDBDriverTest { } @Test - public void testConnectWithProperties() { + public void connectWithProperties() { final String jdbcUrl = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(jdbcUrl, connProps); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); @@ -100,33 +55,21 @@ public class TSDBDriverTest { } @Test - public void testConnectWithConfigFile() { + public void connectWithConfigFile() { String jdbcUrl = "jdbc:TAOS://:/log?user=root&password=taosdata"; Properties connProps = new Properties(); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); try { - if (islibLoaded && isTaosdActived) { - conn = DriverManager.getConnection(jdbcUrl, connProps); - assertNotNull("failure - connection should not be null", conn); - } + conn = DriverManager.getConnection(jdbcUrl, connProps); + assertNotNull("failure - connection should not be null", conn); } catch (SQLException e) { e.printStackTrace(); fail("failure - should not throw Exception"); } } - @Test(expected = SQLException.class) - public void testAcceptsURL() throws SQLException { - Driver driver = new TSDBDriver(); - for (String url : validURLs) { - assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url)); - } - driver.acceptsURL(null); - fail("acceptsURL throws exception when parameter is null"); - } - @Test public void testParseURL() { TSDBDriver driver = new TSDBDriver(); @@ -167,8 +110,19 @@ public class TSDBDriverTest { assertNull("failure - dbname should be null", actual.getProperty("dbname")); } + + @Test(expected = SQLException.class) + public void acceptsURL() throws SQLException { + Driver driver = new TSDBDriver(); + for (String url : validURLs) { + assertTrue("failure - acceptsURL(\" " + url + " \") should be true", driver.acceptsURL(url)); + } + driver.acceptsURL(null); + fail("acceptsURL throws exception when parameter is null"); + } + @Test - public void testGetPropertyInfo() throws SQLException { + public void getPropertyInfo() throws SQLException { Driver driver = new TSDBDriver(); final String url = "jdbc:TAOS://localhost:6030/log?user=root&password=taosdata"; Properties connProps = new Properties(); @@ -188,23 +142,31 @@ public class TSDBDriverTest { } @Test - public void testGetMajorVersion() { - assertEquals("failure - getMajorVersion should be 2", 2, new TSDBDriver().getMajorVersion()); + public void getMajorVersion() { + assertEquals(2, new TSDBDriver().getMajorVersion()); } @Test - public void testGetMinorVersion() { - assertEquals("failure - getMinorVersion should be 0", 0, new TSDBDriver().getMinorVersion()); + public void getMinorVersion() { + assertEquals(0, new TSDBDriver().getMinorVersion()); } @Test - public void testJdbcCompliant() { - assertFalse("failure - jdbcCompliant should be false", new TSDBDriver().jdbcCompliant()); + public void jdbcCompliant() { + assertFalse(new TSDBDriver().jdbcCompliant()); } @Test - public void testGetParentLogger() throws SQLFeatureNotSupportedException { - assertNull("failure - getParentLogger should be be null", new TSDBDriver().getParentLogger()); + public void getParentLogger() throws SQLFeatureNotSupportedException { + assertNull(new TSDBDriver().getParentLogger()); } + @BeforeClass + public static void before() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..161539962da5e26e928a669083954cc0817483a2 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBJNIConnectorTest.java @@ -0,0 +1,89 @@ +package com.taosdata.jdbc; + +import org.junit.Test; + +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.util.ArrayList; +import java.util.List; + +public class TSDBJNIConnectorTest { + + private static TSDBResultSetRowData rowData; + + @Test + public void test() { + try { + // init + TSDBJNIConnector.init("/etc/taos/taos.cfg", null, null, null); + // connect + TSDBJNIConnector connector = new TSDBJNIConnector(); + connector.connect("127.0.0.1", 6030, "unsign_jni", "root", "taosdata"); + // executeQuery + long pSql = connector.executeQuery("select * from unsign_jni.us_table"); + if (connector.isUpdateQuery(pSql)) { + connector.freeResultSet(pSql); + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_INVALID_WITH_EXECUTEQUERY); + } + // get schema + List columnMetaDataList = new ArrayList<>(); + int code = connector.getSchemaMetaData(pSql, columnMetaDataList); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } + if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL); + } + if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0); + } + int columnSize = columnMetaDataList.size(); + // print metadata + for (int i = 0; i < columnSize; i++) { + System.out.println(columnMetaDataList.get(i)); + } + rowData = new TSDBResultSetRowData(columnSize); + // iterate resultSet + for (int i = 0; next(connector, pSql); i++) { + System.out.println("col[" + i + "] size: " + rowData.getColSize()); + rowData.getData().stream().forEach(col -> System.out.print(col + "\t")); + System.out.println(); + } + // close resultSet + code = connector.freeResultSet(pSql); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL); + } + // close statement + + // close connection + connector.closeConnection(); + + } catch (SQLWarning throwables) { + throwables.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private static boolean next(TSDBJNIConnector connector, long pSql) throws SQLException { + if (rowData != null) + rowData.clear(); + + int code = connector.fetchRow(pSql, rowData); + if (code == TSDBConstants.JNI_CONNECTION_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL); + } else if (code == TSDBConstants.JNI_RESULT_SET_NULL) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_RESULT_SET_NULL); + } else if (code == TSDBConstants.JNI_NUM_OF_FIELDS_0) { + throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_JNI_NUM_OF_FIELDS_0); + } else if (code == TSDBConstants.JNI_FETCH_END) { + return false; + } else { + return true; + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..12bcc4391767e129289fae0a945d048570a18bc5 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBParameterMetaDataTest.java @@ -0,0 +1,194 @@ +package com.taosdata.jdbc; + +import com.taosdata.jdbc.rs.RestfulParameterMetaData; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class TSDBParameterMetaDataTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + private static PreparedStatement pstmt_insert; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; + private static PreparedStatement pstmt_select; + private static ParameterMetaData parameterMetaData_insert; + private static ParameterMetaData parameterMetaData_select; + + @Test + public void getParameterCount() throws SQLException { + Assert.assertEquals(10, parameterMetaData_insert.getParameterCount()); + } + + @Test + public void isNullable() throws SQLException { + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(1)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(2)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(3)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(4)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(5)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(6)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(7)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(8)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(9)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(10)); + } + + @Test + public void isSigned() throws SQLException { + Assert.assertEquals(false, parameterMetaData_insert.isSigned(1)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(2)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(3)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(4)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(5)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(6)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(7)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(8)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(9)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(10)); + } + + @Test + public void getPrecision() throws SQLException { + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(1)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(2)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(3)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(4)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(5)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(6)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(7)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(8)); + Assert.assertEquals(5, parameterMetaData_insert.getPrecision(9)); + Assert.assertEquals(5, parameterMetaData_insert.getPrecision(10)); + } + + @Test + public void getScale() throws SQLException { + Assert.assertEquals(0, parameterMetaData_insert.getScale(1)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(2)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(3)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(4)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(5)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(6)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(7)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(8)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(9)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(10)); + } + + @Test + public void getParameterType() throws SQLException { + Assert.assertEquals(Types.TIMESTAMP, parameterMetaData_insert.getParameterType(1)); + Assert.assertEquals(Types.INTEGER, parameterMetaData_insert.getParameterType(2)); + Assert.assertEquals(Types.BIGINT, parameterMetaData_insert.getParameterType(3)); + Assert.assertEquals(Types.FLOAT, parameterMetaData_insert.getParameterType(4)); + Assert.assertEquals(Types.DOUBLE, parameterMetaData_insert.getParameterType(5)); + Assert.assertEquals(Types.SMALLINT, parameterMetaData_insert.getParameterType(6)); + Assert.assertEquals(Types.TINYINT, parameterMetaData_insert.getParameterType(7)); + Assert.assertEquals(Types.BOOLEAN, parameterMetaData_insert.getParameterType(8)); + Assert.assertEquals(Types.BINARY, parameterMetaData_insert.getParameterType(9)); + Assert.assertEquals(Types.NCHAR, parameterMetaData_insert.getParameterType(10)); + } + + @Test + public void getParameterTypeName() throws SQLException { + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.TIMESTAMP), parameterMetaData_insert.getParameterTypeName(1)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.INTEGER), parameterMetaData_insert.getParameterTypeName(2)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BIGINT), parameterMetaData_insert.getParameterTypeName(3)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.FLOAT), parameterMetaData_insert.getParameterTypeName(4)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.DOUBLE), parameterMetaData_insert.getParameterTypeName(5)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.SMALLINT), parameterMetaData_insert.getParameterTypeName(6)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.TINYINT), parameterMetaData_insert.getParameterTypeName(7)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BOOLEAN), parameterMetaData_insert.getParameterTypeName(8)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BINARY), parameterMetaData_insert.getParameterTypeName(9)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.NCHAR), parameterMetaData_insert.getParameterTypeName(10)); + } + + @Test + public void getParameterClassName() throws SQLException { + Assert.assertEquals(Timestamp.class.getName(), parameterMetaData_insert.getParameterClassName(1)); + Assert.assertEquals(Integer.class.getName(), parameterMetaData_insert.getParameterClassName(2)); + Assert.assertEquals(Long.class.getName(), parameterMetaData_insert.getParameterClassName(3)); + Assert.assertEquals(Float.class.getName(), parameterMetaData_insert.getParameterClassName(4)); + Assert.assertEquals(Double.class.getName(), parameterMetaData_insert.getParameterClassName(5)); + Assert.assertEquals(Short.class.getName(), parameterMetaData_insert.getParameterClassName(6)); + Assert.assertEquals(Byte.class.getName(), parameterMetaData_insert.getParameterClassName(7)); + Assert.assertEquals(Boolean.class.getName(), parameterMetaData_insert.getParameterClassName(8)); + Assert.assertEquals(byte[].class.getName(), parameterMetaData_insert.getParameterClassName(9)); + Assert.assertEquals(String.class.getName(), parameterMetaData_insert.getParameterClassName(10)); + } + + @Test + public void getParameterMode() throws SQLException { + for (int i = 1; i <= parameterMetaData_insert.getParameterCount(); i++) { + int parameterMode = parameterMetaData_insert.getParameterMode(i); + Assert.assertEquals(ParameterMetaData.parameterModeUnknown, parameterMode); + } + } + + @Test + public void unwrap() throws SQLException { + TSDBParameterMetaData unwrap = parameterMetaData_insert.unwrap(TSDBParameterMetaData.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(parameterMetaData_insert.isWrapperFor(TSDBParameterMetaData.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_pstmt"); + stmt.execute("create database if not exists test_pstmt"); + stmt.execute("use test_pstmt"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + } + pstmt_insert = conn.prepareStatement(sql_insert); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(2, 111); + pstmt_insert.setObject(3, Long.MAX_VALUE); + pstmt_insert.setObject(4, 3.14159265354f); + pstmt_insert.setObject(5, Double.MAX_VALUE); + pstmt_insert.setObject(6, Short.MAX_VALUE); + pstmt_insert.setObject(7, Byte.MAX_VALUE); + pstmt_insert.setObject(8, true); + pstmt_insert.setObject(9, "hello".getBytes()); + pstmt_insert.setObject(10, "Hello"); + parameterMetaData_insert = pstmt_insert.getParameterMetaData(); + + pstmt_select = conn.prepareStatement(sql_select); + pstmt_select.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_select.setTimestamp(2, new Timestamp(System.currentTimeMillis() + 10000)); + pstmt_select.setInt(3, 0); + parameterMetaData_select = pstmt_select.getParameterMetaData(); + + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (pstmt_insert != null) + pstmt_insert.close(); + if (pstmt_select != null) + pstmt_select.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java new file mode 100644 index 0000000000000000000000000000000000000000..434095efa284b8bae0fafef2795714573e2f76e0 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -0,0 +1,363 @@ +package com.taosdata.jdbc; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.io.Serializable; +import java.sql.*; + +public class TSDBPreparedStatementTest { + private static final String host = "127.0.0.1"; + private static Connection conn; + private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + private static PreparedStatement pstmt_insert; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; + private static PreparedStatement pstmt_select; + + @Test + public void executeQuery() throws SQLException { + long end = System.currentTimeMillis(); + long start = end - 1000 * 60 * 60; + pstmt_select.setTimestamp(1, new Timestamp(start)); + pstmt_select.setTimestamp(2, new Timestamp(end)); + pstmt_select.setInt(3, 0); + + ResultSet rs = pstmt_select.executeQuery(); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + } + + @Test + public void executeUpdate() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setFloat(4, 3.14f); + int result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + } + + @Test + public void setNull() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(2, Types.INTEGER); + int result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + } + + @Test + public void setBoolean() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setBoolean(8, true); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setByte() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setByte(7, (byte) 0x001); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setShort() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setShort(6, (short) 2); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setInt() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setInt(2, 10086); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setLong() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setLong(3, Long.MAX_VALUE); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setFloat() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setFloat(4, 3.14f); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setDouble() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setDouble(5, 3.14444); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBigDecimal() throws SQLException { + pstmt_insert.setBigDecimal(1, null); + } + + @Test + public void setString() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, "aaaa"); + boolean execute = pstmt_insert.execute(); + Assert.assertFalse(execute); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, new Person("john", 33, true).toString()); + Assert.assertFalse(pstmt_insert.execute()); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, new Person("john", 33, true).toString().replaceAll("'", "\"")); + Assert.assertFalse(pstmt_insert.execute()); + } + + class Person implements Serializable { + String name; + int age; + boolean sex; + + public Person(String name, int age, boolean sex) { + this.name = name; + this.age = age; + this.sex = sex; + } + + @Override + public String toString() { + return "Person{" + + "name='" + name + '\'' + + ", age=" + age + + ", sex=" + sex + + '}'; + } + } + + @Test + public void setBytes() throws SQLException, IOException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + +// ByteArrayOutputStream baos = new ByteArrayOutputStream(); +// ObjectOutputStream oos = new ObjectOutputStream(baos); +// oos.writeObject(new Person("john", 33, true)); +// oos.flush(); +// byte[] bytes = baos.toByteArray(); +// pstmt_insert.setBytes(9, bytes); + + pstmt_insert.setBytes(9, new Person("john", 33, true).toString().getBytes()); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setDate() throws SQLException { + pstmt_insert.setDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTime() throws SQLException { + pstmt_insert.setTime(1, new Time(System.currentTimeMillis())); + } + + @Test + public void setTimestamp() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setAsciiStream() throws SQLException { + pstmt_insert.setAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBinaryStream() throws SQLException { + pstmt_insert.setBinaryStream(1, null); + } + + @Test + public void clearParameters() throws SQLException { + pstmt_insert.clearParameters(); + } + + @Test + public void setObject() throws SQLException { + pstmt_insert.setObject(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(2, 111); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(3, Long.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(4, 3.14159265354f); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(5, Double.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(6, Short.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(7, Byte.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(8, true); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(9, "hello".getBytes()); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(10, "Hello"); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void execute() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + executeQuery(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setCharacterStream() throws SQLException { + pstmt_insert.setCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setRef() throws SQLException { + pstmt_insert.setRef(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBlob() throws SQLException { + pstmt_insert.setBlob(1, (Blob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setClob() throws SQLException { + pstmt_insert.setClob(1, (Clob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setArray() throws SQLException { + pstmt_insert.setArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getMetaData() throws SQLException { + pstmt_insert.getMetaData(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setURL() throws SQLException { + pstmt_insert.setURL(1, null); + } + + @Test + public void getParameterMetaData() throws SQLException { + ParameterMetaData parameterMetaData = pstmt_insert.getParameterMetaData(); + Assert.assertNotNull(parameterMetaData); + //TODO: modify the test case + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setRowId() throws SQLException { + pstmt_insert.setRowId(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNString() throws SQLException { + pstmt_insert.setNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNCharacterStream() throws SQLException { + pstmt_insert.setNCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNClob() throws SQLException { + pstmt_insert.setNClob(1, (NClob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setSQLXML() throws SQLException { + pstmt_insert.setSQLXML(1, null); + } + + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_pstmt_jni"); + stmt.execute("create database if not exists test_pstmt_jni"); + stmt.execute("use test_pstmt_jni"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + } + pstmt_insert = conn.prepareStatement(sql_insert); + pstmt_select = conn.prepareStatement(sql_select); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (pstmt_insert != null) + pstmt_insert.close(); + if (pstmt_select != null) + pstmt_select.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java new file mode 100644 index 0000000000000000000000000000000000000000..374b1335fc88e4eefcfd02f90ed5c2e36ede38e1 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBResultSetTest.java @@ -0,0 +1,678 @@ +package com.taosdata.jdbc; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.Shorts; +import com.taosdata.jdbc.rs.RestfulResultSet; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.sql.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +public class TSDBResultSetTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + private static Statement stmt; + private static ResultSet rs; + + @Test + public void wasNull() throws SQLException { + Assert.assertFalse(rs.wasNull()); + } + + @Test + public void getString() throws SQLException { + String f10 = rs.getString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test + public void getBoolean() throws SQLException { + Boolean f9 = rs.getBoolean("f9"); + Assert.assertEquals(true, f9); + f9 = rs.getBoolean(9); + Assert.assertEquals(true, f9); + } + + @Test + public void getByte() throws SQLException { + byte f8 = rs.getByte("f8"); + Assert.assertEquals(10, f8); + f8 = rs.getByte(8); + Assert.assertEquals(10, f8); + } + + @Test + public void getShort() throws SQLException { + short f7 = rs.getShort("f7"); + Assert.assertEquals(10, f7); + f7 = rs.getShort(7); + Assert.assertEquals(10, f7); + } + + @Test + public void getInt() throws SQLException { + int f2 = rs.getInt("f2"); + Assert.assertEquals(1, f2); + f2 = rs.getInt(2); + Assert.assertEquals(1, f2); + } + + @Test + public void getLong() throws SQLException { + long f3 = rs.getLong("f3"); + Assert.assertEquals(100, f3); + f3 = rs.getLong(3); + Assert.assertEquals(100, f3); + } + + @Test + public void getFloat() throws SQLException { + float f4 = rs.getFloat("f4"); + Assert.assertEquals(3.1415f, f4, 0f); + f4 = rs.getFloat(4); + Assert.assertEquals(3.1415f, f4, 0f); + } + + @Test + public void getDouble() throws SQLException { + double f5 = rs.getDouble("f5"); + Assert.assertEquals(3.1415926, f5, 0.0); + f5 = rs.getDouble(5); + Assert.assertEquals(3.1415926, f5, 0.0); + } + + @Test + public void getBigDecimal() throws SQLException { + BigDecimal f1 = rs.getBigDecimal("f1"); + Assert.assertEquals(1609430400000l, f1.longValue()); + + BigDecimal f2 = rs.getBigDecimal("f2"); + Assert.assertEquals(1, f2.intValue()); + + BigDecimal f3 = rs.getBigDecimal("f3"); + Assert.assertEquals(100l, f3.longValue()); + + BigDecimal f4 = rs.getBigDecimal("f4"); + Assert.assertEquals(3.1415f, f4.floatValue(), 0.00000f); + + BigDecimal f5 = rs.getBigDecimal("f5"); + Assert.assertEquals(3.1415926, f5.doubleValue(), 0.0000000); + + BigDecimal f7 = rs.getBigDecimal("f7"); + Assert.assertEquals(10, f7.intValue()); + + BigDecimal f8 = rs.getBigDecimal("f8"); + Assert.assertEquals(10, f8.intValue()); + } + + @Test + public void getBytes() throws SQLException { + byte[] f1 = rs.getBytes("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", new String(f1)); + + byte[] f2 = rs.getBytes("f2"); + Assert.assertEquals(1, Ints.fromByteArray(f2)); + + byte[] f3 = rs.getBytes("f3"); + Assert.assertEquals(100l, Longs.fromByteArray(f3)); + + byte[] f4 = rs.getBytes("f4"); + Assert.assertEquals(3.1415f, Float.valueOf(new String(f4)), 0.000000f); + + byte[] f5 = rs.getBytes("f5"); + Assert.assertEquals(3.1415926, Double.valueOf(new String(f5)), 0.000000f); + + byte[] f6 = rs.getBytes("f6"); + Assert.assertEquals("abc", new String(f6)); + + byte[] f7 = rs.getBytes("f7"); + Assert.assertEquals((short) 10, Shorts.fromByteArray(f7)); + + byte[] f8 = rs.getBytes("f8"); + Assert.assertEquals(1, f8.length); + Assert.assertEquals((byte) 10, f8[0]); + + byte[] f9 = rs.getBytes("f9"); + Assert.assertEquals("true", new String(f9)); + + byte[] f10 = rs.getBytes("f10"); + Assert.assertEquals("涛思数据", new String(f10)); + } + + @Test + public void getDate() throws SQLException, ParseException { + Date f1 = rs.getDate("f1"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Assert.assertEquals(sdf.parse("2021-01-01"), f1); + } + + @Test + public void getTime() throws SQLException { + Time f1 = rs.getTime("f1"); + Assert.assertEquals("00:00:00", f1.toString()); + } + + @Test + public void getTimestamp() throws SQLException { + Timestamp f1 = rs.getTimestamp("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + f1 = rs.getTimestamp(1); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getAsciiStream() throws SQLException { + rs.getAsciiStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getUnicodeStream() throws SQLException { + rs.getUnicodeStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBinaryStream() throws SQLException { + rs.getBinaryStream("f1"); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(rs.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + rs.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCursorName() throws SQLException { + rs.getCursorName(); + } + + @Test + public void getMetaData() throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertNotNull(meta); + } + + @Test + public void getObject() throws SQLException, ParseException { + Object f1 = rs.getObject("f1"); + Assert.assertEquals(Timestamp.class, f1.getClass()); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss.sss"); + java.util.Date date = sdf.parse("2021-01-01 00:00:00.000"); + Assert.assertEquals(new Timestamp(date.getTime()), f1); + + Object f2 = rs.getObject("f2"); + Assert.assertEquals(Integer.class, f2.getClass()); + Assert.assertEquals(1, f2); + + Object f3 = rs.getObject("f3"); + Assert.assertEquals(Long.class, f3.getClass()); + Assert.assertEquals(100l, f3); + + Object f4 = rs.getObject("f4"); + Assert.assertEquals(Float.class, f4.getClass()); + Assert.assertEquals(3.1415f, f4); + + Object f5 = rs.getObject("f5"); + Assert.assertEquals(Double.class, f5.getClass()); + Assert.assertEquals(3.1415926, f5); + + Object f6 = rs.getObject("f6"); + Assert.assertEquals(byte[].class, f6.getClass()); + Assert.assertEquals("abc", new String((byte[]) f6)); + + Object f7 = rs.getObject("f7"); + Assert.assertEquals(Short.class, f7.getClass()); + Assert.assertEquals((short) 10, f7); + + Object f8 = rs.getObject("f8"); + Assert.assertEquals(Byte.class, f8.getClass()); + Assert.assertEquals((byte) 10, f8); + + Object f9 = rs.getObject("f9"); + Assert.assertEquals(Boolean.class, f9.getClass()); + Assert.assertEquals(true, f9); + + Object f10 = rs.getObject("f10"); + Assert.assertEquals(String.class, f10.getClass()); + Assert.assertEquals("涛思数据", f10); + } + + @Test(expected = SQLException.class) + public void findColumn() throws SQLException { + int columnIndex = rs.findColumn("f1"); + Assert.assertEquals(1, columnIndex); + columnIndex = rs.findColumn("f2"); + Assert.assertEquals(2, columnIndex); + columnIndex = rs.findColumn("f3"); + Assert.assertEquals(3, columnIndex); + columnIndex = rs.findColumn("f4"); + Assert.assertEquals(4, columnIndex); + columnIndex = rs.findColumn("f5"); + Assert.assertEquals(5, columnIndex); + columnIndex = rs.findColumn("f6"); + Assert.assertEquals(6, columnIndex); + columnIndex = rs.findColumn("f7"); + Assert.assertEquals(7, columnIndex); + columnIndex = rs.findColumn("f8"); + Assert.assertEquals(8, columnIndex); + columnIndex = rs.findColumn("f9"); + Assert.assertEquals(9, columnIndex); + columnIndex = rs.findColumn("f10"); + Assert.assertEquals(10, columnIndex); + + rs.findColumn("f11"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCharacterStream() throws SQLException { + rs.getCharacterStream(1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void isBeforeFirst() throws SQLException { + rs.isBeforeFirst(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void isAfterLast() throws SQLException { + rs.isAfterLast(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void isFirst() throws SQLException { + rs.isFirst(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void isLast() throws SQLException { + rs.isLast(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void beforeFirst() throws SQLException { + rs.beforeFirst(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void afterLast() throws SQLException { + rs.afterLast(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void first() throws SQLException { + rs.first(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void last() throws SQLException { + rs.last(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRow() throws SQLException { + int row = rs.getRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void absolute() throws SQLException { + rs.absolute(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void relative() throws SQLException { + rs.relative(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void previous() throws SQLException { + rs.previous(); + } + + @Test + public void setFetchDirection() throws SQLException { + rs.setFetchDirection(ResultSet.FETCH_FORWARD); + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + rs.setFetchDirection(ResultSet.FETCH_UNKNOWN); + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test + public void setFetchSize() throws SQLException { + rs.setFetchSize(0); + Assert.assertEquals(0, rs.getFetchSize()); + } + + @Test + public void getFetchSize() throws SQLException { + Assert.assertEquals(0, rs.getFetchSize()); + } + + @Test + public void getType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, rs.getType()); + } + + @Test + public void getConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, rs.getConcurrency()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowUpdated() throws SQLException { + rs.rowUpdated(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowInserted() throws SQLException { + rs.rowInserted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowDeleted() throws SQLException { + rs.rowDeleted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNull() throws SQLException { + rs.updateNull("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBoolean() throws SQLException { + rs.updateBoolean(1, false); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateByte() throws SQLException { + rs.updateByte(1, new Byte("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateShort() throws SQLException { + rs.updateShort(1, new Short("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateInt() throws SQLException { + rs.updateInt(1, 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateLong() throws SQLException { + rs.updateLong(1, 1l); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateFloat() throws SQLException { + rs.updateFloat(1, 1f); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDouble() throws SQLException { + rs.updateDouble(1, 1.0); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBigDecimal() throws SQLException { + rs.updateBigDecimal(1, new BigDecimal(1)); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateString() throws SQLException { + rs.updateString(1, "abc"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBytes() throws SQLException { + rs.updateBytes(1, new byte[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDate() throws SQLException { + rs.updateDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTime() throws SQLException { + rs.updateTime(1, new Time(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTimestamp() throws SQLException { + rs.updateTimestamp(1, new Timestamp(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateAsciiStream() throws SQLException { + rs.updateAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBinaryStream() throws SQLException { + rs.updateBinaryStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateCharacterStream() throws SQLException { + rs.updateCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateObject() throws SQLException { + rs.updateObject(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void insertRow() throws SQLException { + rs.insertRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRow() throws SQLException { + rs.updateRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void deleteRow() throws SQLException { + rs.deleteRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void refreshRow() throws SQLException { + rs.refreshRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancelRowUpdates() throws SQLException { + rs.cancelRowUpdates(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToInsertRow() throws SQLException { + rs.moveToInsertRow(); + } + + @Test + public void getStatement() throws SQLException { + Statement stmt = rs.getStatement(); + Assert.assertNotNull(stmt); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToCurrentRow() throws SQLException { + rs.moveToCurrentRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRef() throws SQLException { + rs.getRef(1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBlob() throws SQLException { + rs.getBlob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getClob() throws SQLException { + rs.getClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getArray() throws SQLException { + rs.getArray("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getURL() throws SQLException { + rs.getURL("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRef() throws SQLException { + rs.updateRef("f1", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBlob() throws SQLException { + rs.updateBlob(1, (InputStream) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateClob() throws SQLException { + rs.updateClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateArray() throws SQLException { + rs.updateArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRowId() throws SQLException { + rs.getRowId("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRowId() throws SQLException { + rs.updateRowId(1, null); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, rs.getHoldability()); + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(rs.isClosed()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNString() throws SQLException { + rs.updateNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNClob() throws SQLException { + rs.updateNClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNClob() throws SQLException { + rs.getNClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getSQLXML() throws SQLException { + rs.getSQLXML("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateSQLXML() throws SQLException { + rs.updateSQLXML(1, null); + } + + @Test + public void getNString() throws SQLException { + String f10 = rs.getNString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getNString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNCharacterStream() throws SQLException { + rs.getNCharacterStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNCharacterStream() throws SQLException { + rs.updateNCharacterStream(1, null); + } + + @Test + public void unwrap() throws SQLException { + TSDBResultSet unwrap = rs.unwrap(TSDBResultSet.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(rs.isWrapperFor(TSDBResultSet.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + + } + + @AfterClass + public static void afterClass() { + try { + if (rs != null) + rs.close(); + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java new file mode 100644 index 0000000000000000000000000000000000000000..671551c42639620b09ace1ae315da6f25ea278d4 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBStatementTest.java @@ -0,0 +1,415 @@ +package com.taosdata.jdbc; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; +import java.util.UUID; + +public class TSDBStatementTest { + private static final String host = "127.0.0.1"; + private static Connection conn; + private static Statement stmt; + + @Test + public void executeQuery() { + try { + ResultSet rs = stmt.executeQuery("show databases"); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void executeUpdate() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + int affectRows = stmt.executeUpdate("create database " + dbName); + Assert.assertEquals(0, affectRows); + affectRows = stmt.executeUpdate("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(0, affectRows); + affectRows = stmt.executeUpdate("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(1, affectRows); + affectRows = stmt.executeUpdate("drop database " + dbName); + Assert.assertEquals(0, affectRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void close() { + // test in AfterClass method + } + + + @Test + public void getMaxFieldSize() throws SQLException { + Assert.assertEquals(16 * 1024, stmt.getMaxFieldSize()); + } + + @Test(expected = SQLException.class) + public void setMaxFieldSize() throws SQLException { + + stmt.setMaxFieldSize(0); + stmt.setMaxFieldSize(-1); + } + + @Test + public void getMaxRows() throws SQLException { + Assert.assertEquals(0, stmt.getMaxRows()); + } + + @Test(expected = SQLException.class) + public void setMaxRows() throws SQLException { + stmt.setMaxRows(0); + stmt.setMaxRows(-1); + } + + @Test + public void setEscapeProcessing() throws SQLException { + stmt.setEscapeProcessing(true); + stmt.setEscapeProcessing(false); + } + + @Test + public void getQueryTimeout() throws SQLException { + Assert.assertEquals(0, stmt.getQueryTimeout()); + } + + @Test(expected = SQLException.class) + public void setQueryTimeout() throws SQLException { + stmt.setQueryTimeout(0); + stmt.setQueryTimeout(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancel() throws SQLException { + stmt.cancel(); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(stmt.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + stmt.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setCursorName() throws SQLException { + stmt.setCursorName(""); + } + + @Test + public void execute() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + boolean isSelect = stmt.execute("create database " + dbName); + Assert.assertEquals(false, isSelect); + int affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(1, affectedRows); + + isSelect = stmt.execute("select * from " + dbName + ".weather"); + Assert.assertEquals(true, isSelect); + + isSelect = stmt.execute("drop database " + dbName); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getResultSet() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + boolean isSelect = stmt.execute("create database " + dbName); + Assert.assertEquals(false, isSelect); + int affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(1, affectedRows); + + isSelect = stmt.execute("select * from " + dbName + ".weather"); + Assert.assertEquals(true, isSelect); + ResultSet rs = stmt.getResultSet(); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertEquals(3, meta.getColumnCount()); + int count = 0; + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + count++; + } + Assert.assertEquals(1, count); + + isSelect = stmt.execute("drop database " + dbName); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getUpdateCount() { + // already test in execute method + } + + @Test + public void getMoreResults() throws SQLException { + Assert.assertEquals(false, stmt.getMoreResults()); + } + + @Test(expected = SQLException.class) + public void setFetchDirection() throws SQLException { + stmt.setFetchDirection(ResultSet.FETCH_FORWARD); + stmt.setFetchDirection(ResultSet.FETCH_REVERSE); + stmt.setFetchDirection(ResultSet.FETCH_UNKNOWN); + stmt.setFetchDirection(-1); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, stmt.getFetchDirection()); + } + + @Test(expected = SQLException.class) + public void setFetchSize() throws SQLException { + stmt.setFetchSize(0); + stmt.setFetchSize(-1); + } + + @Test + public void getFetchSize() throws SQLException { + stmt.setFetchSize(0); + Assert.assertEquals(0, stmt.getFetchSize()); + stmt.setFetchSize(0); + } + + @Test + public void getResultSetConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, stmt.getResultSetConcurrency()); + } + + @Test + public void getResultSetType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, stmt.getResultSetType()); + } + + @Test + public void addBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void clearBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.clearBatch(); + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + stmt.clearBatch(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void executeBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + int[] results = stmt.executeBatch(); + Assert.assertEquals(0, results[0]); + Assert.assertEquals(0, results[1]); + Assert.assertEquals(1, results[2]); + Assert.assertEquals(Statement.SUCCESS_NO_INFO, results[3]); + Assert.assertEquals(0, results[4]); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getConnection() { + try { + Connection connection = stmt.getConnection(); + Assert.assertNotNull(connection); + Assert.assertTrue(this.conn == connection); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testGetMoreResults() throws SQLException { + Assert.assertEquals(false, stmt.getMoreResults(Statement.CLOSE_CURRENT_RESULT)); + stmt.getMoreResults(Statement.KEEP_CURRENT_RESULT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getGeneratedKeys() throws SQLException { + stmt.getGeneratedKeys(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate() throws SQLException { + stmt.executeUpdate("", 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate1() throws SQLException { + stmt.executeUpdate("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate2() throws SQLException { + stmt.executeUpdate("", new String[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute() throws SQLException { + stmt.execute("", 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute1() throws SQLException { + stmt.execute("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute2() throws SQLException { + stmt.execute("", new String[]{}); + } + + @Test + public void getResultSetHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, stmt.getResultSetHoldability()); + } + + @Test + public void isClosed() { + try { + Assert.assertEquals(false, stmt.isClosed()); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void setPoolable() throws SQLException { + stmt.setPoolable(true); + stmt.setPoolable(false); + } + + @Test + public void isPoolable() throws SQLException { + Assert.assertEquals(false, stmt.isPoolable()); + } + + @Test + public void closeOnCompletion() throws SQLException { + stmt.closeOnCompletion(); + } + + @Test + public void isCloseOnCompletion() throws SQLException { + Assert.assertFalse(stmt.isCloseOnCompletion()); + } + + @Test + public void unwrap() throws SQLException { + TSDBStatement unwrap = stmt.unwrap(TSDBStatement.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(stmt.isWrapperFor(TSDBStatement.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata", properties); + stmt = conn.createStatement(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java index 9608c4985db15312e8e2966badfe4db8241bc306..472da34980d98cbe694783b120bc5533a908fa5b 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchInsertTest.java @@ -1,14 +1,12 @@ package com.taosdata.jdbc.cases; -import com.taosdata.jdbc.lib.TSDBCommon; +import com.taosdata.jdbc.TSDBDriver; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; +import java.util.Properties; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -18,57 +16,71 @@ import static org.junit.Assert.assertEquals; public class BatchInsertTest { - static String host = "localhost"; + static String host = "127.0.0.1"; static String dbName = "test"; static String stbName = "meters"; static int numOfTables = 30; final static int numOfRecordsPerTable = 1000; static long ts = 1496732686000l; final static String tablePrefix = "t"; - private Connection connection; @Before public void before() { try { - connection = TSDBCommon.getConn(host); - TSDBCommon.createDatabase(connection, dbName); - TSDBCommon.createStable(connection, stbName); - TSDBCommon.createTables(connection, numOfTables, stbName, tablePrefix); + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + Statement statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + statement.executeUpdate("create database if not exists " + dbName); + statement.executeUpdate("use " + dbName); + // create stable + String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; + statement.executeUpdate(createTableSql); + // create tables + for(int i = 0; i < numOfTables; i++) { + String loc = i % 2 == 0 ? "beijing" : "shanghai"; + String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; + statement.executeUpdate(createSubTalbesSql); + } + statement.close(); } catch (Exception e) { e.printStackTrace(); } } @Test - public void testBatchInsert(){ + public void testBatchInsert() { ExecutorService executorService = Executors.newFixedThreadPool(numOfTables); for (int i = 0; i < numOfTables; i++) { final int index = i; - executorService.execute(new Runnable() { - @Override - public void run() { - try { - long startTime = System.currentTimeMillis(); - Statement statement = connection.createStatement(); // get statement - StringBuilder sb = new StringBuilder(); - sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); - Random rand = new Random(); - for (int j = 1; j <= numOfRecordsPerTable; j++) { - sb.append("(" + (ts + j) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ", "); - sb.append(rand.nextInt(100) + ")"); - } - statement.addBatch(sb.toString()); - statement.executeBatch(); - long endTime = System.currentTimeMillis(); - System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); - connection.commit(); - statement.close(); - } catch (Exception e) { - e.printStackTrace(); + executorService.execute(() -> { + try { + long startTime = System.currentTimeMillis(); + Statement statement = connection.createStatement(); // get statement + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO " + tablePrefix + index + " VALUES"); + Random rand = new Random(); + for (int j = 1; j <= numOfRecordsPerTable; j++) { + sb.append("(" + (ts + j) + ", "); + sb.append(rand.nextInt(100) + ", "); + sb.append(rand.nextInt(100) + ", "); + sb.append(rand.nextInt(100) + ")"); } + statement.addBatch(sb.toString()); + statement.executeBatch(); + long endTime = System.currentTimeMillis(); + System.out.println("Thread " + index + " takes " + (endTime - startTime) + " microseconds"); + connection.commit(); + statement.close(); + } catch (Exception e) { + e.printStackTrace(); } }); } @@ -80,7 +92,7 @@ public class BatchInsertTest { e.printStackTrace(); } - try{ + try { Statement statement = connection.createStatement(); ResultSet rs = statement.executeQuery("select * from meters"); int num = 0; @@ -89,7 +101,7 @@ public class BatchInsertTest { } assertEquals(num, numOfTables * numOfRecordsPerTable); rs.close(); - }catch (Exception e){ + } catch (Exception e) { e.printStackTrace(); } } @@ -102,7 +114,6 @@ public class BatchInsertTest { } catch (SQLException e) { e.printStackTrace(); } - } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java index e4d2d7598d6a6bd5610abcb1bd576f523cb22740..d8d8f1eae42f555b2bc0be12e8a3a943071da3c6 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ConnectWrongDatabaseTest.java @@ -1,5 +1,6 @@ package com.taosdata.jdbc.cases; +import com.taosdata.jdbc.TSDBErrorNumbers; import org.junit.Assert; import org.junit.Test; @@ -16,8 +17,7 @@ public class ConnectWrongDatabaseTest { } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (SQLException e) { - System.out.println(e.getMessage()); - Assert.assertEquals("TDengine Error: Invalid database name", e.getMessage()); + Assert.assertEquals(TSDBErrorNumbers.ERROR_JNI_CONNECTION_NULL, e.getErrorCode()); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java new file mode 100644 index 0000000000000000000000000000000000000000..f97e555ad1b1acc7b6dd0024d893fcc1ccd4cc53 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DatetimeBefore1970Test.java @@ -0,0 +1,68 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.utils.TimestampUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class DatetimeBefore1970Test { + + private static Connection conn; + + @Test + public void test() { + try (Statement stmt = conn.createStatement()) { + stmt.executeUpdate("insert into weather(ts) values('1969-12-31 23:59:59.999')"); + stmt.executeUpdate("insert into weather(ts) values('1970-01-01 00:00:00.000')"); + stmt.executeUpdate("insert into weather(ts) values('1970-01-01 08:00:00.000')"); + stmt.executeUpdate("insert into weather(ts) values('1970-01-01 07:59:59.999')"); + + ResultSet rs = stmt.executeQuery("select * from weather"); + while (rs.next()) { + Timestamp ts = rs.getTimestamp("ts"); + System.out.println("long: " + ts.getTime() + ", string: " + TimestampUtil.longToDatetime(ts.getTime())); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + public static void main(String[] args) { + System.out.println("timestamp: " + Long.MAX_VALUE + ", string: " + TimestampUtil.longToDatetime(Long.MAX_VALUE)); + System.out.println("timestamp: " + Long.MIN_VALUE + ", string: " + TimestampUtil.longToDatetime(Long.MIN_VALUE)); + System.out.println("timestamp: " + 0 + ", string: " + TimestampUtil.longToDatetime(0)); + System.out.println("timestamp: " + -1 + ", string: " + TimestampUtil.longToDatetime(-1)); + String datetime = "1970-01-01 00:00:00.000"; + System.out.println("timestamp: " + TimestampUtil.datetimeToLong(datetime) + ", string: " + datetime); + datetime = "1969-12-31 23:59:59.999"; + System.out.println("timestamp: " + TimestampUtil.datetimeToLong(datetime) + ", string: " + datetime); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists test_timestamp"); + stmt.execute("create database if not exists test_timestamp keep 36500"); + stmt.execute("use test_timestamp"); + stmt.execute("create table weather(ts timestamp,f1 float)"); + stmt.close(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java new file mode 100644 index 0000000000000000000000000000000000000000..9826e6ed764f8319c70a8a2c1edf7b2444b4f21d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java @@ -0,0 +1,41 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +public class DriverAutoloadTest { + + private Properties properties; + private String host = "127.0.0.1"; + + @Test + public void testRestful() throws SQLException { + final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + Connection conn = DriverManager.getConnection(url, properties); + Assert.assertNotNull(conn); + } + + @Test + public void testJni() throws SQLException { + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + Connection conn = DriverManager.getConnection(url, properties); + Assert.assertNotNull(conn); + } + + + @Before + public void before() { + properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java index 83295df5274a669ca2fc7fdbba506a97a01cc55c..be2ff94e02e10c14f1229b63762f5a6af7082318 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/FailOverTest.java @@ -4,7 +4,6 @@ import org.junit.Test; import java.sql.*; import java.text.SimpleDateFormat; -import java.util.Date; import java.util.concurrent.TimeUnit; public class FailOverTest { @@ -18,13 +17,17 @@ public class FailOverTest { long end = System.currentTimeMillis() + 1000 * 60 * 5; while (System.currentTimeMillis() < end) { - try (Connection conn = DriverManager.getConnection(url)) { - Statement stmt = conn.createStatement(); - ResultSet resultSet = stmt.executeQuery("select server_status()"); - resultSet.next(); - int status = resultSet.getInt("server_status()"); - System.out.println(">>>>>>>>>" + sdf.format(new Date()) + " status : " + status); - stmt.close(); + try (Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("show dnodes"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + System.out.println("======================="); + rs.close(); TimeUnit.SECONDS.sleep(5); } catch (SQLException | InterruptedException e) { e.printStackTrace(); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java new file mode 100644 index 0000000000000000000000000000000000000000..d7603312a090bedb17ed125edf6da535924964d0 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/ImportTest.java @@ -0,0 +1,113 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.*; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class ImportTest { + private static Connection connection; + static String dbName = "test_import"; + static String tName = "t0"; + static String host = "127.0.0.1"; + private static long ts; + + @BeforeClass + public static void before() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + Statement stmt = connection.createStatement(); + stmt.execute("create database if not exists " + dbName); + stmt.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); + stmt.close(); + + ts = System.currentTimeMillis(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @Test + public void case001_insertData() throws Exception { + try (Statement stmt = connection.createStatement()) { + for (int i = 0; i < 50; i++) { + ts++; + int row = stmt.executeUpdate("import into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); + System.out.println("import into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); + assertEquals(1, row); + } + } + } + + @Test + public void case002_checkSum() { + Assert.assertEquals(50, select()); + } + + private int select() { + int count = 0; + try (Statement stmt = connection.createStatement()) { + + String sql = "select * from " + dbName + "." + tName; + ResultSet rs = stmt.executeQuery(sql); + while (rs.next()) { + count++; + } + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + return count; + } + + @Test + public void case003_importData() { + // 避免时间重复 + try (Statement stmt = connection.createStatement()) { + StringBuilder sqlBuilder = new StringBuilder("import into ").append(dbName).append(".").append(tName).append(" values "); + for (int i = 0; i < 50; i++) { + int a = i / 5; + long t = (++ts) + a; + sqlBuilder.append("(").append(t).append(",").append((100 + i)).append(",").append(i).append(") "); + } + System.out.println(sqlBuilder.toString()); + int rows = stmt.executeUpdate(sqlBuilder.toString()); + assertEquals(50, rows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void case004_checkSum() { + Assert.assertEquals(100, select()); + } + + @AfterClass + public static void close() { + try { + if (connection != null) { + Statement statement = connection.createStatement(); + statement.executeUpdate("drop database " + dbName); + statement.close(); + connection.close(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java new file mode 100644 index 0000000000000000000000000000000000000000..84149775c3d2bbda6a66930159e19eb27d95a6d1 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertDbwithoutUseDbTest.java @@ -0,0 +1,91 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Properties; +import java.util.Random; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class InsertDbwithoutUseDbTest { + + private static String host = "127.0.0.1"; + private static Properties properties; + private static Random random = new Random(System.currentTimeMillis()); + + @Test + public void case001() throws ClassNotFoundException, SQLException { + // prepare schema + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata"; + Connection conn = DriverManager.getConnection(url, properties); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists inWithoutDb"); + stmt.execute("create database if not exists inWithoutDb"); + stmt.execute("create table inWithoutDb.weather(ts timestamp, f1 int)"); + } + conn.close(); + + // execute insert + url = "jdbc:TAOS://127.0.0.1:6030/inWithoutDb?user=root&password=taosdata"; + conn = DriverManager.getConnection(url, properties); + try (Statement stmt = conn.createStatement()) { + int affectedRow = stmt.executeUpdate("insert into weather(ts, f1) values(now," + random.nextInt(100) + ")"); + Assert.assertEquals(1, affectedRow); + boolean flag = stmt.execute("insert into weather(ts, f1) values(now + 10s," + random.nextInt(100) + ")"); + Assert.assertEquals(false, flag); + ResultSet rs = stmt.executeQuery("select count(*) from weather"); + rs.next(); + int count = rs.getInt("count(*)"); + Assert.assertEquals(2, count); + + } catch (SQLException e) { + e.printStackTrace(); + } + + conn.close(); + } + + @Test + public void case002() throws ClassNotFoundException, SQLException { + // prepare the schema + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + final String url = "jdbc:TAOS-RS://" + host + ":6041/inWithoutDb?user=root&password=taosdata"; + Connection conn = DriverManager.getConnection(url, properties); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists inWithoutDb"); + stmt.execute("create database if not exists inWithoutDb"); + stmt.execute("create table inWithoutDb.weather(ts timestamp, f1 int)"); + } + conn.close(); + + // execute + conn = DriverManager.getConnection(url, properties); + try (Statement stmt = conn.createStatement()) { + int affectedRow = stmt.executeUpdate("insert into weather(ts, f1) values(now," + random.nextInt(100) + ")"); + Assert.assertEquals(1, affectedRow); + boolean flag = stmt.execute("insert into weather(ts, f1) values(now + 10s," + random.nextInt(100) + ")"); + Assert.assertEquals(false, flag); + ResultSet rs = stmt.executeQuery("select count(*) from weather"); + rs.next(); + int count = rs.getInt("count(*)"); + Assert.assertEquals(2, count); + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() { + properties = new Properties(); + properties.setProperty("charset", "UTF-8"); + properties.setProperty("locale", "en_US.UTF-8"); + properties.setProperty("timezone", "UTC-8"); + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f3d79b1df1594edc4fffd626244bc742ea13ec75 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InvalidResultSetPointerTest.java @@ -0,0 +1,187 @@ +package com.taosdata.jdbc.cases; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +public class InvalidResultSetPointerTest { + + private static String host = "127.0.0.1"; + private static final String dbName = "test"; + private static final String stbName = "stb"; + private static final String tbName = "tb"; + private static Connection connection; + private static int numOfSTb = 30000; + private static int numOfTb = 3; + private static int numOfThreads = 100; + + @Test + public void test() throws SQLException { + execute("drop database if exists " + dbName); + execute("create database if not exists " + dbName); + execute("use " + dbName); + createSTable(); + createTable(); + insert(); + selectMultiThreading(); + close(); + } + + private void insert() { + for (int i = 0; i < numOfSTb; i++) { + for (int j = 0; j < numOfTb; j++) { + final String sql = "INSERT INTO " + dbName + "." + tbName + i + "_" + j + " (ts, temperature, humidity, name) values(now, 20.5, 34, \"" + i + "\")"; + System.out.println(sql); + execute(sql); + } + } + } + + private void createSTable() { + for (int i = 0; i < numOfSTb; i++) { + final String sql = "create table if not exists " + dbName + "." + stbName + i + " (ts timestamp, temperature float, humidity int, name BINARY(" + (i % 73 + 10) + ")) TAGS (tag1 INT)"; + execute(sql); + } + } + + private void createTable() { + for (int i = 0; i < numOfSTb; i++) { + for (int j = 0; j < numOfTb; j++) { + final String sql = "create table if not exists " + dbName + "." + tbName + i + "_" + j + " USING " + stbName + i + " TAGS(" + j + ")"; + execute(sql); + } + } + } + + private void close() throws SQLException { + if (connection != null) { + this.connection.close(); + System.out.println("connection closed."); + } + } + + private void selectMultiThreading() { + int a = numOfSTb / numOfThreads; + if (a < 1) { + numOfThreads = numOfSTb; + a = 1; + } + + int b = 0; + if (numOfThreads != 0) { + b = numOfSTb % numOfThreads; + } + + multiThreadingClass instance[] = new multiThreadingClass[numOfThreads]; + + int last = 0; + for (int i = 0; i < numOfThreads; i++) { + instance[i] = new multiThreadingClass(); + instance[i].id = i; + instance[i].from = last; + if (i < b) { + instance[i].to = last + a; + } else { + instance[i].to = last + a - 1; + } + + last = instance[i].to + 1; + instance[i].numOfTb = numOfTb; + instance[i].connection = connection; + instance[i].dbName = dbName; + instance[i].tbName = tbName; + instance[i].start(); + } + + for (int i = 0; i < numOfThreads; i++) { + try { + instance[i].join(); + } catch (InterruptedException ie) { + ie.printStackTrace(); + } + } + } + + @BeforeClass + public static void beforeClass() { + try { + String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty("charset", "UTF-8"); + properties.setProperty("locale", "en_US.UTF-8"); + properties.setProperty("timezone", "UTC-8"); + System.out.println("get connection starting..."); + connection = DriverManager.getConnection(url, properties); + if (connection != null) + System.out.println("[ OK ] Connection established."); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + private void execute(String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + statement.execute(sql); + long end = System.currentTimeMillis(); + printSql(sql, (end - start)); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private void printSql(String sql, long cost) { + System.out.println("time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private void executeQuery(String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + ResultSet resultSet = statement.executeQuery(sql); + long end = System.currentTimeMillis(); + printSql(sql, (end - start)); +// printResult(resultSet); + resultSet.close(); + statement.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + class multiThreadingClass extends Thread { + public int id; + public int from, to; + public int numOfTb; + public Connection connection; + public String dbName; + public String tbName; + + public void run() { + System.out.println("ID: " + id + " from: " + from + " to: " + to); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + System.out.println("Thread " + id + " interrupted."); + } + + for (int i = from; i < to; i++) { + for (int j = 0; j < numOfTb; j++) { + if (j % 1000 == 0) { + try { + System.out.print(id + "s."); + Thread.sleep(1); + } catch (InterruptedException e) { + System.out.println("Thread " + id + " interrupted."); + } + } + final String sql = "select last_row(humidity) from " + dbName + "." + tbName + i + "_" + j; + executeQuery(sql); + } + } + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java new file mode 100644 index 0000000000000000000000000000000000000000..5cb76cc0cb71e1cb879b696d05cc6682f93d0bdc --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/MultiThreadsWithSameStatmentTest.java @@ -0,0 +1,91 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.concurrent.TimeUnit; + +public class MultiThreadsWithSameStatmentTest { + + + private class Service { + public Connection conn; + public Statement stmt; + + public Service() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists jdbctest"); + stmt.executeUpdate("create table if not exists jdbctest.weather (ts timestamp, f1 int)"); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + public void release() { + try { + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + + @Before + public void before() { + } + + @Test + public void test() { + Thread t1 = new Thread(() -> { + try { + Service service = new Service(); + ResultSet resultSet = service.stmt.executeQuery("select * from jdbctest.weather"); + while (resultSet.next()) { + ResultSetMetaData metaData = resultSet.getMetaData(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + System.out.print(metaData.getColumnLabel(i) + ": " + resultSet.getString(i)); + } + System.out.println(); + } + resultSet.close(); + service.release(); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + + Thread t2 = new Thread(() -> { + while (true) { + try { + Service service = new Service(); + service.stmt.executeUpdate("insert into jdbctest.weather values(now,1)"); + service.release(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + }); + t1.start(); + sleep(1000); + t2.start(); + } + + private void sleep(long mills) { + try { + TimeUnit.MILLISECONDS.sleep(mills); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f2ac94adc1b6e7d94e52650dcfbb5664c8f39760 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java @@ -0,0 +1,64 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; + +public class NullValueInResultSetForJdbcRestfulTest { + + private static final String host = "127.0.0.1"; + Connection conn; + + @Test + public void test() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + Object value = rs.getObject(i); + System.out.print(meta.getColumnLabel(i) + ": " + value + "\t"); + } + System.out.println(); + } + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Before + public void before() throws SQLException { + final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_null"); + stmt.execute("create database if not exists test_null"); + stmt.execute("use test_null"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64))"); + stmt.executeUpdate("insert into weather(ts, f1) values(now+1s, 1)"); + stmt.executeUpdate("insert into weather(ts, f2) values(now+2s, 2)"); + stmt.executeUpdate("insert into weather(ts, f3) values(now+3s, 3.0)"); + stmt.executeUpdate("insert into weather(ts, f4) values(now+4s, 4.0)"); + stmt.executeUpdate("insert into weather(ts, f5) values(now+5s, 5)"); + stmt.executeUpdate("insert into weather(ts, f6) values(now+6s, 6)"); + stmt.executeUpdate("insert into weather(ts, f7) values(now+7s, true)"); + stmt.executeUpdate("insert into weather(ts, f8) values(now+8s, 'hello')"); + stmt.executeUpdate("insert into weather(ts, f9) values(now+9s, '涛思数据')"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..d0ba113b7a4a8f99e22eb8143905d0b086583e1d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/QueryDataTest.java @@ -0,0 +1,73 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +public class QueryDataTest { + + static Connection connection; + static Statement statement; + static String dbName = "test"; + static String stbName = "meters"; + static String host = "127.0.0.1"; + + @Before + public void createDatabase() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + statement = connection.createStatement(); + statement.executeUpdate("drop database if exists " + dbName); + statement.executeUpdate("create database if not exists " + dbName); + statement.executeUpdate("use " + dbName); + + String createTableSql = "create table " + stbName + "(ts timestamp, name binary(64))"; + statement.executeUpdate(createTableSql); + + } catch (ClassNotFoundException | SQLException e) { + return; + } + } + + @Test + public void testQueryBinaryData() throws SQLException { + String insertSql = "insert into " + stbName + " values(now, 'taosdata')"; + System.out.println(insertSql); + statement.executeUpdate(insertSql); + + String querySql = "select * from " + stbName; + ResultSet rs = statement.executeQuery(querySql); + + while (rs.next()) { + String name = rs.getString(2); + System.out.println("name = " + name); + assertEquals("taosdata", name); + } + rs.close(); + } + + @After + public void close() { + try { + if (statement != null) + statement.close(); + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java new file mode 100644 index 0000000000000000000000000000000000000000..38c8cbb98c48342f131f4f5f0fee885bb446e83c --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/SelectTest.java @@ -0,0 +1,82 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +public class SelectTest { + Connection connection; + String dbName = "test"; + String tName = "t0"; + String host = "127.0.0.1"; + + @Before + public void createDatabaseAndTable() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + + Statement stmt = connection.createStatement(); + stmt.execute("drop database if exists " + dbName); + stmt.execute("create database if not exists " + dbName); + stmt.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); + stmt.close(); + } catch (ClassNotFoundException e) { + return; + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @Test + public void selectData() { + long ts = 1496732686000l; + + try (Statement stmt = connection.createStatement()) { + for (int i = 0; i < 50; i++) { + ts++; + int row = stmt.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")"); + System.out.println("insert into " + dbName + "." + tName + " values (" + ts + ", " + (100 + i) + ", " + i + ")\t" + row); + assertEquals(1, row); + } + + String sql = "select * from " + dbName + "." + tName; + ResultSet resSet = stmt.executeQuery(sql); + + int num = 0; + while (resSet.next()) { + num++; + } + resSet.close(); + assertEquals(num, 50); + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @After + public void close() { + try { + if (connection != null) { + Statement stmt = connection.createStatement(); + stmt.executeUpdate("drop database " + dbName); + stmt.close(); + connection.close(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java new file mode 100644 index 0000000000000000000000000000000000000000..4575cb73a05fbbc19d6eaf2ba5be0ed27b61804c --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/StableTest.java @@ -0,0 +1,111 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class StableTest { + + private static Connection connection; + private static String dbName = "test"; + private static String stbName = "st"; + private static String host = "127.0.0.1"; + + @BeforeClass + public static void createDatabase() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); + Statement statement = connection.createStatement(); + statement.execute("drop database if exists " + dbName); + statement.execute("create database if not exists " + dbName); + statement.execute("use " + dbName); + statement.close(); + } catch (ClassNotFoundException e) { + return; + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void case001_createSuperTable() { + try (Statement stmt = connection.createStatement()) { + final String sql = "create table " + stbName + " (ts timestamp, v1 int, v2 int) tags (tg nchar(20)) "; + stmt.execute(sql); + } catch (SQLException e) { + assert false : "error create stable" + e.getMessage(); + } + } + + @Test + public void case002_createTable() { + try (Statement stmt = connection.createStatement()) { + final String sql = "create table t1 using " + stbName + " tags (\"beijing\")"; + stmt.execute(sql); + } catch (SQLException e) { + assert false : "error create table" + e.getMessage(); + } + } + + @Test + public void case003_describeSTable() { + int num = 0; + try (Statement stmt = connection.createStatement()) { + String sql = "describe " + stbName; + ResultSet rs = stmt.executeQuery(sql); + while (rs.next()) { + for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) { + System.out.println(i + ":" + rs.getString(i)); + } + num++; + } + rs.close(); + assertEquals(4, num); + } catch (SQLException e) { + assert false : "error describe stable" + e.getMessage(); + } + } + + @Test + public void case004_describeTable() { + int num = 0; + try (Statement stmt = connection.createStatement()) { + ResultSet rs = stmt.executeQuery("describe t1"); + while (rs.next()) { + for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) { + System.out.printf("%d: %s\n", i, rs.getString(i)); + } + num++; + } + rs.close(); + assertEquals(4, num); + } catch (SQLException e) { + assert false : "error describe stable" + e.getMessage(); + } + } + + @AfterClass + public static void close() { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..e9e36e20c4f83e98d6dd808da3a58ee628f418f0 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java @@ -0,0 +1,49 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Test; + +import java.sql.*; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class TaosInfoMonitorTest { + + @Test + public void testCreateTooManyConnection() throws ClassNotFoundException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + final String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata"; + + List connectionList = IntStream.range(0, 100).mapToObj(i -> { + try { + TimeUnit.MILLISECONDS.sleep(100); + return DriverManager.getConnection(url); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + return null; + }).collect(Collectors.toList()); + + connectionList.stream().forEach(conn -> { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("show databases"); + while (rs.next()) { + + } + TimeUnit.MILLISECONDS.sleep(100); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + }); + + connectionList.stream().forEach(conn -> { + try { + conn.close(); + TimeUnit.MILLISECONDS.sleep(100); + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + }); + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java new file mode 100644 index 0000000000000000000000000000000000000000..d1816a3e7c84703e76a28e6b0e5db842de9c0ac4 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberJniTest.java @@ -0,0 +1,191 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.*; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Properties; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class UnsignedNumberJniTest { + private static final String host = "127.0.0.1"; + private static Connection conn; + + @Test + public void testCase001() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from us_table"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + Assert.assertEquals("127", rs.getString(2)); + Assert.assertEquals("32767", rs.getString(3)); + Assert.assertEquals("2147483647", rs.getString(4)); + Assert.assertEquals("9223372036854775807", rs.getString(5)); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase002() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from us_table"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals(2147483647, rs.getInt(4)); + Assert.assertEquals(9223372036854775807l, rs.getLong(5)); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test(expected = SQLException.class) + public void testCase003() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,2147483647, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + Assert.assertEquals(2147483647, rs.getInt(4)); + } + } + } + + @Test(expected = SQLException.class) + public void testCase004() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,4294967294, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + Assert.assertEquals(127, rs.getByte(2)); + Assert.assertEquals(32767, rs.getShort(3)); + } + } + } + + @Test(expected = SQLException.class) + public void testCase005() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 65534,4294967294, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + + Assert.assertEquals(127, rs.getByte(2)); + } + } + } + + @Test(expected = SQLException.class) + public void testCase006() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + } + } + } + + @Test + public void testCase007() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + Assert.assertEquals("254", rs.getString(2)); + Assert.assertEquals("65534", rs.getString(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + } + } + } + + + @BeforeClass + public static void beforeClass() { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url, properties); + + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists unsign_jni"); + stmt.execute("create database if not exists unsign_jni"); + stmt.execute("use unsign_jni"); + stmt.execute("create table us_table(ts timestamp, f1 tinyint unsigned, f2 smallint unsigned, f3 int unsigned, f4 bigint unsigned)"); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(now, 127, 32767,2147483647, 9223372036854775807)"); + stmt.close(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java new file mode 100644 index 0000000000000000000000000000000000000000..4ae2f36fe95a37eba27dd78222f3120f5cbfaf3a --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/UnsignedNumberRestfulTest.java @@ -0,0 +1,177 @@ +package com.taosdata.jdbc.cases; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.*; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Properties; + + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class UnsignedNumberRestfulTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + + @Test + public void testCase001() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from us_table"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase002() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from us_table"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test(expected = SQLException.class) + public void testCase003() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,2147483647, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + } + } + } + + @Test(expected = SQLException.class) + public void testCase004() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 32767,4294967294, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + } + } + } + + @Test(expected = SQLException.class) + public void testCase005() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 127, 65534,4294967294, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + } + } + } + + @Test(expected = SQLException.class) + public void testCase006() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + rs.getTimestamp(1) + "\t"); + System.out.print(meta.getColumnLabel(2) + ": " + rs.getByte(2) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + rs.getShort(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + rs.getInt(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + rs.getLong(5) + "\t"); + System.out.println(); + } + } + } + + @Test + public void testCase007() throws SQLException { + try (Statement stmt = conn.createStatement()) { + long now = System.currentTimeMillis(); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(" + now + ", 254, 65534,4294967294, 18446744073709551614)"); + ResultSet rs = stmt.executeQuery("select * from us_table where ts = " + now); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + Assert.assertEquals("254", rs.getString(2)); + Assert.assertEquals("65534", rs.getString(3)); + Assert.assertEquals("4294967294", rs.getString(4)); + Assert.assertEquals("18446744073709551614", rs.getString(5)); + } + } + } + + @BeforeClass + public static void beforeClass() { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url, properties); + + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists unsign_restful"); + stmt.execute("create database if not exists unsign_restful"); + stmt.execute("use unsign_restful"); + stmt.execute("create table us_table(ts timestamp, f1 tinyint unsigned, f2 smallint unsigned, f3 int unsigned, f4 bigint unsigned)"); + stmt.executeUpdate("insert into us_table(ts,f1,f2,f3,f4) values(now, 127, 32767,2147483647, 9223372036854775807)"); + stmt.close(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java deleted file mode 100644 index 0e2613d617074571c86c39ef253d68af9a0f6922..0000000000000000000000000000000000000000 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/lib/TSDBCommon.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.taosdata.jdbc.lib; - -import com.taosdata.jdbc.TSDBDriver; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Properties; - -public class TSDBCommon { - - public static Connection getConn(String host) throws SQLException, ClassNotFoundException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - Properties properties = new Properties(); - properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); - properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - return DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); - } - - public static void createDatabase(Connection connection, String dbName) throws SQLException { - Statement statement = connection.createStatement(); - statement.executeUpdate("drop database if exists " + dbName); - statement.executeUpdate("create database if not exists " + dbName); - statement.executeUpdate("use " + dbName); - statement.close(); - } - - public static void createStable(Connection connection, String stbName) throws SQLException { - Statement statement = connection.createStatement(); - String createTableSql = "create table " + stbName + "(ts timestamp, f1 int, f2 int, f3 int) tags(areaid int, loc binary(20))"; - statement.executeUpdate(createTableSql); - statement.close(); - } - - public static void createTables(Connection connection, int numOfTables, String stbName,String tablePrefix) throws SQLException { - Statement statement = connection.createStatement(); - for(int i = 0; i < numOfTables; i++) { - String loc = i % 2 == 0 ? "beijing" : "shanghai"; - String createSubTalbesSql = "create table " + tablePrefix + i + " using " + stbName + " tags(" + i + ", '" + loc + "')"; - statement.executeUpdate(createSubTalbesSql); - } - statement.close(); - } -} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a6fb6cfda044b4e88c5bd5509c51d114507d84f7 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/AuthenticationTest.java @@ -0,0 +1,65 @@ +package com.taosdata.jdbc.rs; + +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; + +public class AuthenticationTest { + + private static final String host = "127.0.0.1"; + private static final String user = "root"; + private static final String password = "taos?data"; + private Connection conn; + + @Test + public void test() { + // change password + try { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("alter user " + user + " pass '" + password + "'"); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + // use new to login and execute query + try { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=" + password); + Statement stmt = conn.createStatement(); + stmt.execute("show databases"); + ResultSet rs = stmt.getResultSet(); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ":" + rs.getString(i) + "\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + // change password back + try { + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=" + user + "&password=" + password); + Statement stmt = conn.createStatement(); + stmt.execute("alter user " + user + " pass 'taosdata'"); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + + } + + @Before + public void before() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java new file mode 100644 index 0000000000000000000000000000000000000000..abd60f5b63d46b406f19b6be9dcbbab6b786de12 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulConnectionTest.java @@ -0,0 +1,402 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +public class RestfulConnectionTest { + + private static final String host = "127.0.0.1"; + + private static Connection conn; + + @Test + public void getConnection() { + // already test in beforeClass method + } + + @Test + public void createStatement() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void prepareStatement() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()"); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void prepareCall() throws SQLException { + conn.prepareCall("select server_status()"); + } + + @Test + public void nativeSQL() throws SQLException { + String nativeSQL = conn.nativeSQL("select * from log.log"); + Assert.assertEquals("select * from log.log", nativeSQL); + } + + @Test + public void setAutoCommit() throws SQLException { + conn.setAutoCommit(true); + conn.setAutoCommit(false); + } + + @Test + public void getAutoCommit() throws SQLException { + Assert.assertTrue(conn.getAutoCommit()); + } + + @Test + public void commit() throws SQLException { + conn.commit(); + } + + @Test + public void rollback() throws SQLException { + conn.rollback(); + } + + @Test + public void close() { + // connection will close in afterClass method + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(conn.isClosed()); + } + + @Test + public void getMetaData() throws SQLException { + DatabaseMetaData meta = conn.getMetaData(); + Assert.assertNotNull(meta); + Assert.assertEquals("com.taosdata.jdbc.rs.RestfulDriver", meta.getDriverName()); + } + + @Test + public void setReadOnly() throws SQLException { + conn.setReadOnly(true); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertTrue(conn.isReadOnly()); + } + + @Test + public void setCatalog() throws SQLException { + conn.setCatalog("test"); + Assert.assertEquals("test", conn.getCatalog()); + } + + @Test + public void getCatalog() throws SQLException { + conn.setCatalog("log"); + Assert.assertEquals("log", conn.getCatalog()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTransactionIsolation() throws SQLException { + conn.setTransactionIsolation(Connection.TRANSACTION_NONE); + Assert.assertEquals(Connection.TRANSACTION_NONE, conn.getTransactionIsolation()); + conn.setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); + } + + @Test + public void getTransactionIsolation() throws SQLException { + Assert.assertEquals(Connection.TRANSACTION_NONE, conn.getTransactionIsolation()); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(conn.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + conn.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testCreateStatement() throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()", + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.prepareStatement("select server_status", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareCall() throws SQLException { + conn.prepareCall("", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getTypeMap() throws SQLException { + conn.getTypeMap(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTypeMap() throws SQLException { + conn.setTypeMap(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setHoldability() throws SQLException { + conn.setHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT); + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, conn.getHoldability()); + conn.setHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, conn.getHoldability()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setSavepoint() throws SQLException { + conn.setSavepoint(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testSetSavepoint() throws SQLException { + conn.setSavepoint(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testRollback() throws SQLException { + conn.rollback(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void releaseSavepoint() throws SQLException { + conn.releaseSavepoint(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testCreateStatement1() throws SQLException { + Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + ResultSet rs = stmt.executeQuery("select server_status()"); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement1() throws SQLException { + PreparedStatement pstmt = conn.prepareStatement("select server_status()", + ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + ResultSet rs = pstmt.executeQuery(); + rs.next(); + int status = rs.getInt("server_status()"); + Assert.assertEquals(1, status); + + conn.prepareStatement("select server_status", ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareCall1() throws SQLException { + conn.prepareCall("", ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.HOLD_CURSORS_OVER_COMMIT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement2() throws SQLException { + Assert.assertNotNull("", Statement.NO_GENERATED_KEYS); + conn.prepareStatement("", Statement.RETURN_GENERATED_KEYS); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement3() throws SQLException { + conn.prepareStatement("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testPrepareStatement4() throws SQLException { + conn.prepareStatement("", new String[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createClob() throws SQLException { + conn.createClob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createBlob() throws SQLException { + conn.createBlob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createNClob() throws SQLException { + conn.createNClob(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createSQLXML() throws SQLException { + conn.createSQLXML(); + } + + @Test(expected = SQLException.class) + public void isValid() throws SQLException { + Assert.assertTrue(conn.isValid(10)); + Assert.assertTrue(conn.isValid(0)); + conn.isValid(-1); + } + + @Test + public void setClientInfo() throws SQLClientInfoException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTC-8"); + } + + @Test + public void testSetClientInfo() throws SQLClientInfoException { + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn.setClientInfo(properties); + } + + @Test + public void getClientInfo() throws SQLException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + Properties info = conn.getClientInfo(); + String charset = info.getProperty(TSDBDriver.PROPERTY_KEY_CHARSET); + Assert.assertEquals("UTF-8", charset); + String locale = info.getProperty(TSDBDriver.PROPERTY_KEY_LOCALE); + Assert.assertEquals("en_US.UTF-8", locale); + String timezone = info.getProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE); + Assert.assertEquals("UTC-8", timezone); + } + + @Test + public void testGetClientInfo() throws SQLException { + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + conn.setClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + + String charset = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_CHARSET); + Assert.assertEquals("UTF-8", charset); + String locale = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_LOCALE); + Assert.assertEquals("en_US.UTF-8", locale); + String timezone = conn.getClientInfo(TSDBDriver.PROPERTY_KEY_TIME_ZONE); + Assert.assertEquals("UTC-8", timezone); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createArrayOf() throws SQLException { + conn.createArrayOf("", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void createStruct() throws SQLException { + conn.createStruct("", null); + } + + @Test + public void setSchema() throws SQLException { + conn.setSchema("test"); + } + + @Test + public void getSchema() throws SQLException { + Assert.assertNull(conn.getSchema()); + } + + @Test + public void abort() throws SQLException { + conn.abort(null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNetworkTimeout() throws SQLException { + conn.setNetworkTimeout(null, 1000); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNetworkTimeout() throws SQLException { + conn.getNetworkTimeout(); + } + + @Test + public void unwrap() { + try { + RestfulConnection restfulConnection = conn.unwrap(RestfulConnection.class); + Assert.assertNotNull(restfulConnection); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(conn.isWrapperFor(RestfulConnection.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/log?user=root&password=taosdata", properties); + // create test database for test cases + try (Statement stmt = conn.createStatement()) { + stmt.execute("create database if not exists test"); + } + + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a052fbbdcbc241a18cb7dd73b8b4ade053533541 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDatabaseMetaDataTest.java @@ -0,0 +1,983 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; + +public class RestfulDatabaseMetaDataTest { + private static final String host = "127.0.0.1"; + private static final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + private static Connection connection; + private static RestfulDatabaseMetaData metaData; + + @Test + public void unwrap() throws SQLException { + RestfulDatabaseMetaData unwrap = metaData.unwrap(RestfulDatabaseMetaData.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(metaData.isWrapperFor(RestfulDatabaseMetaData.class)); + } + + @Test + public void allProceduresAreCallable() throws SQLException { + Assert.assertFalse(metaData.allProceduresAreCallable()); + } + + @Test + public void allTablesAreSelectable() throws SQLException { + Assert.assertFalse(metaData.allTablesAreSelectable()); + } + + @Test + public void getURL() throws SQLException { + Assert.assertEquals(url, metaData.getURL()); + } + + @Test + public void getUserName() throws SQLException { + Assert.assertEquals("root", metaData.getUserName()); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertFalse(metaData.isReadOnly()); + } + + @Test + public void nullsAreSortedHigh() throws SQLException { + Assert.assertFalse(metaData.nullsAreSortedHigh()); + } + + @Test + public void nullsAreSortedLow() throws SQLException { + Assert.assertTrue(metaData.nullsAreSortedLow()); + } + + @Test + public void nullsAreSortedAtStart() throws SQLException { + Assert.assertTrue(metaData.nullsAreSortedAtStart()); + } + + @Test + public void nullsAreSortedAtEnd() throws SQLException { + Assert.assertFalse(metaData.nullsAreSortedAtEnd()); + } + + @Test + public void getDatabaseProductName() throws SQLException { + Assert.assertEquals("TDengine", metaData.getDatabaseProductName()); + } + + @Test + public void getDatabaseProductVersion() throws SQLException { + Assert.assertEquals("2.0.x.x", metaData.getDatabaseProductVersion()); + } + + @Test + public void getDriverName() throws SQLException { + Assert.assertEquals("com.taosdata.jdbc.rs.RestfulDriver", metaData.getDriverName()); + } + + @Test + public void getDriverVersion() throws SQLException { + Assert.assertEquals("2.0.x", metaData.getDriverVersion()); + } + + @Test + public void getDriverMajorVersion() { + Assert.assertEquals(2, metaData.getDriverMajorVersion()); + } + + @Test + public void getDriverMinorVersion() { + Assert.assertEquals(0, metaData.getDriverMinorVersion()); + } + + @Test + public void usesLocalFiles() throws SQLException { + Assert.assertFalse(metaData.usesLocalFiles()); + } + + @Test + public void usesLocalFilePerTable() throws SQLException { + Assert.assertFalse(metaData.usesLocalFilePerTable()); + } + + @Test + public void supportsMixedCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.supportsMixedCaseIdentifiers()); + } + + @Test + public void storesUpperCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesUpperCaseIdentifiers()); + } + + @Test + public void storesLowerCaseIdentifiers() throws SQLException { + Assert.assertTrue(metaData.storesLowerCaseIdentifiers()); + } + + @Test + public void storesMixedCaseIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesMixedCaseIdentifiers()); + } + + @Test + public void supportsMixedCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.supportsMixedCaseQuotedIdentifiers()); + } + + @Test + public void storesUpperCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesUpperCaseQuotedIdentifiers()); + } + + @Test + public void storesLowerCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesLowerCaseQuotedIdentifiers()); + } + + @Test + public void storesMixedCaseQuotedIdentifiers() throws SQLException { + Assert.assertFalse(metaData.storesMixedCaseQuotedIdentifiers()); + } + + @Test + public void getIdentifierQuoteString() throws SQLException { + Assert.assertEquals(" ", metaData.getIdentifierQuoteString()); + } + + @Test + public void getSQLKeywords() throws SQLException { + Assert.assertEquals(null, metaData.getSQLKeywords()); + } + + @Test + public void getNumericFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getNumericFunctions()); + } + + @Test + public void getStringFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getStringFunctions()); + } + + @Test + public void getSystemFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getSystemFunctions()); + } + + @Test + public void getTimeDateFunctions() throws SQLException { + Assert.assertEquals(null, metaData.getTimeDateFunctions()); + } + + @Test + public void getSearchStringEscape() throws SQLException { + Assert.assertEquals(null, metaData.getSearchStringEscape()); + } + + @Test + public void getExtraNameCharacters() throws SQLException { + Assert.assertEquals(null, metaData.getExtraNameCharacters()); + } + + @Test + public void supportsAlterTableWithAddColumn() throws SQLException { + Assert.assertTrue(metaData.supportsAlterTableWithAddColumn()); + } + + @Test + public void supportsAlterTableWithDropColumn() throws SQLException { + Assert.assertTrue(metaData.supportsAlterTableWithDropColumn()); + } + + @Test + public void supportsColumnAliasing() throws SQLException { + Assert.assertTrue(metaData.supportsColumnAliasing()); + } + + @Test + public void nullPlusNonNullIsNull() throws SQLException { + Assert.assertFalse(metaData.nullPlusNonNullIsNull()); + } + + @Test + public void supportsConvert() throws SQLException { + Assert.assertFalse(metaData.supportsConvert()); + } + + @Test + public void testSupportsConvert() throws SQLException { + Assert.assertFalse(metaData.supportsConvert(1, 1)); + } + + @Test + public void supportsTableCorrelationNames() throws SQLException { + Assert.assertFalse(metaData.supportsTableCorrelationNames()); + } + + @Test + public void supportsDifferentTableCorrelationNames() throws SQLException { + Assert.assertFalse(metaData.supportsDifferentTableCorrelationNames()); + } + + @Test + public void supportsExpressionsInOrderBy() throws SQLException { + Assert.assertFalse(metaData.supportsExpressionsInOrderBy()); + } + + @Test + public void supportsOrderByUnrelated() throws SQLException { + Assert.assertFalse(metaData.supportsOrderByUnrelated()); + } + + @Test + public void supportsGroupBy() throws SQLException { + Assert.assertTrue(metaData.supportsGroupBy()); + } + + @Test + public void supportsGroupByUnrelated() throws SQLException { + Assert.assertFalse(metaData.supportsGroupByUnrelated()); + } + + @Test + public void supportsGroupByBeyondSelect() throws SQLException { + Assert.assertFalse(metaData.supportsGroupByBeyondSelect()); + } + + @Test + public void supportsLikeEscapeClause() throws SQLException { + Assert.assertFalse(metaData.supportsLikeEscapeClause()); + } + + @Test + public void supportsMultipleResultSets() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleResultSets()); + } + + @Test + public void supportsMultipleTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleTransactions()); + } + + @Test + public void supportsNonNullableColumns() throws SQLException { + Assert.assertFalse(metaData.supportsNonNullableColumns()); + } + + @Test + public void supportsMinimumSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsMinimumSQLGrammar()); + } + + @Test + public void supportsCoreSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsCoreSQLGrammar()); + } + + @Test + public void supportsExtendedSQLGrammar() throws SQLException { + Assert.assertFalse(metaData.supportsExtendedSQLGrammar()); + } + + @Test + public void supportsANSI92EntryLevelSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92EntryLevelSQL()); + } + + @Test + public void supportsANSI92IntermediateSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92IntermediateSQL()); + } + + @Test + public void supportsANSI92FullSQL() throws SQLException { + Assert.assertFalse(metaData.supportsANSI92FullSQL()); + } + + @Test + public void supportsIntegrityEnhancementFacility() throws SQLException { + Assert.assertFalse(metaData.supportsIntegrityEnhancementFacility()); + } + + @Test + public void supportsOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsOuterJoins()); + } + + @Test + public void supportsFullOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsFullOuterJoins()); + } + + @Test + public void supportsLimitedOuterJoins() throws SQLException { + Assert.assertFalse(metaData.supportsLimitedOuterJoins()); + } + + @Test + public void getSchemaTerm() throws SQLException { + Assert.assertNull(metaData.getSchemaTerm()); + } + + @Test + public void getProcedureTerm() throws SQLException { + Assert.assertNull(metaData.getProcedureTerm()); + } + + @Test + public void getCatalogTerm() throws SQLException { + Assert.assertEquals("database", metaData.getCatalogTerm()); + } + + @Test + public void isCatalogAtStart() throws SQLException { + Assert.assertTrue(metaData.isCatalogAtStart()); + } + + @Test + public void getCatalogSeparator() throws SQLException { + Assert.assertEquals(".", metaData.getCatalogSeparator()); + } + + @Test + public void supportsSchemasInDataManipulation() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInDataManipulation()); + } + + @Test + public void supportsSchemasInProcedureCalls() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInProcedureCalls()); + } + + @Test + public void supportsSchemasInTableDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInTableDefinitions()); + } + + @Test + public void supportsSchemasInIndexDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInIndexDefinitions()); + } + + @Test + public void supportsSchemasInPrivilegeDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsSchemasInPrivilegeDefinitions()); + } + + @Test + public void supportsCatalogsInDataManipulation() throws SQLException { + Assert.assertTrue(metaData.supportsCatalogsInDataManipulation()); + } + + @Test + public void supportsCatalogsInProcedureCalls() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInProcedureCalls()); + } + + @Test + public void supportsCatalogsInTableDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInTableDefinitions()); + } + + @Test + public void supportsCatalogsInIndexDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInIndexDefinitions()); + } + + @Test + public void supportsCatalogsInPrivilegeDefinitions() throws SQLException { + Assert.assertFalse(metaData.supportsCatalogsInPrivilegeDefinitions()); + } + + @Test + public void supportsPositionedDelete() throws SQLException { + Assert.assertFalse(metaData.supportsPositionedDelete()); + } + + @Test + public void supportsPositionedUpdate() throws SQLException { + Assert.assertFalse(metaData.supportsPositionedUpdate()); + } + + @Test + public void supportsSelectForUpdate() throws SQLException { + Assert.assertFalse(metaData.supportsSelectForUpdate()); + } + + @Test + public void supportsStoredProcedures() throws SQLException { + Assert.assertFalse(metaData.supportsStoredProcedures()); + } + + @Test + public void supportsSubqueriesInComparisons() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInComparisons()); + } + + @Test + public void supportsSubqueriesInExists() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInExists()); + } + + @Test + public void supportsSubqueriesInIns() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInIns()); + } + + @Test + public void supportsSubqueriesInQuantifieds() throws SQLException { + Assert.assertFalse(metaData.supportsSubqueriesInQuantifieds()); + } + + @Test + public void supportsCorrelatedSubqueries() throws SQLException { + Assert.assertFalse(metaData.supportsCorrelatedSubqueries()); + } + + @Test + public void supportsUnion() throws SQLException { + Assert.assertFalse(metaData.supportsUnion()); + } + + @Test + public void supportsUnionAll() throws SQLException { + Assert.assertFalse(metaData.supportsUnionAll()); + } + + @Test + public void supportsOpenCursorsAcrossCommit() throws SQLException { + Assert.assertFalse(metaData.supportsOpenCursorsAcrossCommit()); + } + + @Test + public void supportsOpenCursorsAcrossRollback() throws SQLException { + Assert.assertFalse(metaData.supportsOpenCursorsAcrossRollback()); + } + + @Test + public void supportsOpenStatementsAcrossCommit() throws SQLException { + Assert.assertFalse(metaData.supportsOpenStatementsAcrossCommit()); + } + + @Test + public void supportsOpenStatementsAcrossRollback() throws SQLException { + Assert.assertFalse(metaData.supportsOpenStatementsAcrossRollback()); + } + + @Test + public void getMaxBinaryLiteralLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxBinaryLiteralLength()); + } + + @Test + public void getMaxCharLiteralLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCharLiteralLength()); + } + + @Test + public void getMaxColumnNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnNameLength()); + } + + @Test + public void getMaxColumnsInGroupBy() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInGroupBy()); + } + + @Test + public void getMaxColumnsInIndex() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInIndex()); + } + + @Test + public void getMaxColumnsInOrderBy() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInOrderBy()); + } + + @Test + public void getMaxColumnsInSelect() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInSelect()); + } + + @Test + public void getMaxColumnsInTable() throws SQLException { + Assert.assertEquals(0, metaData.getMaxColumnsInTable()); + } + + @Test + public void getMaxConnections() throws SQLException { + Assert.assertEquals(0, metaData.getMaxConnections()); + } + + @Test + public void getMaxCursorNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCursorNameLength()); + } + + @Test + public void getMaxIndexLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxIndexLength()); + } + + @Test + public void getMaxSchemaNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxSchemaNameLength()); + } + + @Test + public void getMaxProcedureNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxProcedureNameLength()); + } + + @Test + public void getMaxCatalogNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxCatalogNameLength()); + } + + @Test + public void getMaxRowSize() throws SQLException { + Assert.assertEquals(0, metaData.getMaxRowSize()); + } + + @Test + public void doesMaxRowSizeIncludeBlobs() throws SQLException { + Assert.assertFalse(metaData.doesMaxRowSizeIncludeBlobs()); + } + + @Test + public void getMaxStatementLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxStatementLength()); + } + + @Test + public void getMaxStatements() throws SQLException { + Assert.assertEquals(0, metaData.getMaxStatements()); + } + + @Test + public void getMaxTableNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxTableNameLength()); + } + + @Test + public void getMaxTablesInSelect() throws SQLException { + Assert.assertEquals(0, metaData.getMaxTablesInSelect()); + } + + @Test + public void getMaxUserNameLength() throws SQLException { + Assert.assertEquals(0, metaData.getMaxUserNameLength()); + } + + @Test + public void getDefaultTransactionIsolation() throws SQLException { + Assert.assertEquals(Connection.TRANSACTION_NONE, metaData.getDefaultTransactionIsolation()); + } + + @Test + public void supportsTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsTransactions()); + } + + @Test + public void supportsTransactionIsolationLevel() throws SQLException { + Assert.assertTrue(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_NONE)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_COMMITTED)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_READ_UNCOMMITTED)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_REPEATABLE_READ)); + Assert.assertFalse(metaData.supportsTransactionIsolationLevel(Connection.TRANSACTION_SERIALIZABLE)); + } + + @Test + public void supportsDataDefinitionAndDataManipulationTransactions() throws SQLException { + Assert.assertFalse(metaData.supportsDataDefinitionAndDataManipulationTransactions()); + } + + @Test + public void supportsDataManipulationTransactionsOnly() throws SQLException { + Assert.assertFalse(metaData.supportsDataManipulationTransactionsOnly()); + } + + @Test + public void dataDefinitionCausesTransactionCommit() throws SQLException { + Assert.assertFalse(metaData.dataDefinitionCausesTransactionCommit()); + } + + @Test + public void dataDefinitionIgnoredInTransactions() throws SQLException { + Assert.assertFalse(metaData.dataDefinitionIgnoredInTransactions()); + } + + @Test + public void getProcedures() throws SQLException { + Assert.assertNull(metaData.getProcedures("*", "*", "*")); + } + + @Test + public void getProcedureColumns() throws SQLException { + Assert.assertNull(metaData.getProcedureColumns("*", "*", "*", "*")); + } + + @Test + public void getTables() throws SQLException { + System.out.println("****************************************************"); + ResultSet tables = metaData.getTables("log", "", null, null); + ResultSetMetaData metaData = tables.getMetaData(); + while (tables.next()) { + System.out.print(metaData.getColumnLabel(1) + ":" + tables.getString(1) + "\t"); + System.out.print(metaData.getColumnLabel(3) + ":" + tables.getString(3) + "\t"); + System.out.print(metaData.getColumnLabel(4) + ":" + tables.getString(4) + "\t"); + System.out.print(metaData.getColumnLabel(5) + ":" + tables.getString(5) + "\n"); + } + System.out.println(); + Assert.assertNotNull(tables); + } + + @Test + public void getSchemas() throws SQLException { + Assert.assertNotNull(metaData.getSchemas()); + } + + @Test + public void getCatalogs() throws SQLException { + System.out.println("****************************************************"); + + ResultSet catalogs = metaData.getCatalogs(); + ResultSetMetaData meta = catalogs.getMetaData(); + while (catalogs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + catalogs.getString(i)); + } + System.out.println(); + } + } + + @Test + public void getTableTypes() throws SQLException { + System.out.println("****************************************************"); + + ResultSet tableTypes = metaData.getTableTypes(); + while (tableTypes.next()) { + System.out.println(tableTypes.getString("TABLE_TYPE")); + } + Assert.assertNotNull(metaData.getTableTypes()); + } + + @Test + public void getColumns() throws SQLException { + System.out.println("****************************************************"); + + ResultSet columns = metaData.getColumns("log", "", "dn", ""); + ResultSetMetaData meta = columns.getMetaData(); + while (columns.next()) { + System.out.print(meta.getColumnLabel(1) + ": " + columns.getString(1) + "\t"); + System.out.print(meta.getColumnLabel(3) + ": " + columns.getString(3) + "\t"); + System.out.print(meta.getColumnLabel(4) + ": " + columns.getString(4) + "\t"); + System.out.print(meta.getColumnLabel(5) + ": " + columns.getString(5) + "\t"); + System.out.print(meta.getColumnLabel(6) + ": " + columns.getString(6) + "\t"); + System.out.print(meta.getColumnLabel(7) + ": " + columns.getString(7) + "\t"); + System.out.print(meta.getColumnLabel(9) + ": " + columns.getString(9) + "\t"); + System.out.print(meta.getColumnLabel(10) + ": " + columns.getString(10) + "\t"); + System.out.print(meta.getColumnLabel(11) + ": " + columns.getString(11) + "\n"); + System.out.print(meta.getColumnLabel(12) + ": " + columns.getString(12) + "\n"); + } + } + + @Test + public void getColumnPrivileges() throws SQLException { + Assert.assertNotNull(metaData.getColumnPrivileges("", "", "", "")); + } + + @Test + public void getTablePrivileges() throws SQLException { + Assert.assertNotNull(metaData.getTablePrivileges("", "", "")); + } + + @Test + public void getBestRowIdentifier() throws SQLException { + Assert.assertNotNull(metaData.getBestRowIdentifier("", "", "", 0, false)); + } + + @Test + public void getVersionColumns() throws SQLException { + Assert.assertNotNull(metaData.getVersionColumns("", "", "")); + } + + @Test + public void getPrimaryKeys() throws SQLException { + System.out.println("****************************************************"); + + ResultSet rs = metaData.getPrimaryKeys("log", "", "dn1"); + while (rs.next()) { + System.out.println("TABLE_NAME: " + rs.getString("TABLE_NAME")); + System.out.println("COLUMN_NAME: " + rs.getString("COLUMN_NAME")); + System.out.println("KEY_SEQ: " + rs.getString("KEY_SEQ")); + System.out.println("PK_NAME: " + rs.getString("PK_NAME")); + } + + Assert.assertNotNull(rs); + } + + @Test + public void getImportedKeys() throws SQLException { + Assert.assertNotNull(metaData.getImportedKeys("", "", "")); + } + + @Test + public void getExportedKeys() throws SQLException { + Assert.assertNotNull(metaData.getExportedKeys("", "", "")); + } + + @Test + public void getCrossReference() throws SQLException { + Assert.assertNotNull(metaData.getCrossReference("", "", "", "", "", "")); + } + + @Test + public void getTypeInfo() throws SQLException { + Assert.assertNotNull(metaData.getTypeInfo()); + } + + @Test + public void getIndexInfo() throws SQLException { + Assert.assertNotNull(metaData.getIndexInfo("", "", "", false, false)); + } + + @Test + public void supportsResultSetType() throws SQLException { + Assert.assertFalse(metaData.supportsResultSetType(0)); + } + + @Test + public void supportsResultSetConcurrency() throws SQLException { + Assert.assertFalse(metaData.supportsResultSetConcurrency(0, 0)); + } + + @Test + public void ownUpdatesAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownUpdatesAreVisible(0)); + } + + @Test + public void ownDeletesAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownDeletesAreVisible(0)); + } + + @Test + public void ownInsertsAreVisible() throws SQLException { + Assert.assertFalse(metaData.ownInsertsAreVisible(0)); + } + + @Test + public void othersUpdatesAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersUpdatesAreVisible(0)); + } + + @Test + public void othersDeletesAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersDeletesAreVisible(0)); + } + + @Test + public void othersInsertsAreVisible() throws SQLException { + Assert.assertFalse(metaData.othersInsertsAreVisible(0)); + } + + @Test + public void updatesAreDetected() throws SQLException { + Assert.assertFalse(metaData.updatesAreDetected(0)); + } + + @Test + public void deletesAreDetected() throws SQLException { + Assert.assertFalse(metaData.deletesAreDetected(0)); + } + + @Test + public void insertsAreDetected() throws SQLException { + Assert.assertFalse(metaData.insertsAreDetected(0)); + } + + @Test + public void supportsBatchUpdates() throws SQLException { + Assert.assertFalse(metaData.supportsBatchUpdates()); + } + + @Test + public void getUDTs() throws SQLException { + Assert.assertNotNull(metaData.getUDTs("", "", "", null)); + } + + @Test + public void getConnection() throws SQLException { + Assert.assertNotNull(metaData.getConnection()); + } + + @Test + public void supportsSavepoints() throws SQLException { + Assert.assertFalse(metaData.supportsSavepoints()); + } + + @Test + public void supportsNamedParameters() throws SQLException { + Assert.assertFalse(metaData.supportsNamedParameters()); + } + + @Test + public void supportsMultipleOpenResults() throws SQLException { + Assert.assertFalse(metaData.supportsMultipleOpenResults()); + } + + @Test + public void supportsGetGeneratedKeys() throws SQLException { + Assert.assertFalse(metaData.supportsGetGeneratedKeys()); + } + + @Test + public void getSuperTypes() throws SQLException { + Assert.assertNotNull(metaData.getSuperTypes("", "", "")); + } + + @Test + public void getSuperTables() throws SQLException { + System.out.println("****************************************************"); + + ResultSet rs = metaData.getSuperTables("log", "", "dn1"); + while (rs.next()) { + System.out.println("TABLE_NAME: " + rs.getString("TABLE_NAME")); + System.out.println("SUPERTABLE_NAME: " + rs.getString("SUPERTABLE_NAME")); + } + Assert.assertNotNull(rs); + } + + @Test + public void getAttributes() throws SQLException { + Assert.assertNotNull(metaData.getAttributes("", "", "", "")); + } + + @Test + public void supportsResultSetHoldability() throws SQLException { + Assert.assertTrue(metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT)); + Assert.assertFalse(metaData.supportsResultSetHoldability(ResultSet.CLOSE_CURSORS_AT_COMMIT)); + } + + @Test + public void getResultSetHoldability() throws SQLException { + Assert.assertEquals(1, metaData.getResultSetHoldability()); + } + + @Test + public void getDatabaseMajorVersion() throws SQLException { + Assert.assertEquals(2, metaData.getDatabaseMajorVersion()); + } + + @Test + public void getDatabaseMinorVersion() throws SQLException { + Assert.assertEquals(0, metaData.getDatabaseMinorVersion()); + } + + @Test + public void getJDBCMajorVersion() throws SQLException { + Assert.assertEquals(2, metaData.getJDBCMajorVersion()); + } + + @Test + public void getJDBCMinorVersion() throws SQLException { + Assert.assertEquals(0, metaData.getJDBCMinorVersion()); + } + + @Test + public void getSQLStateType() throws SQLException { + Assert.assertEquals(0, metaData.getSQLStateType()); + } + + @Test + public void locatorsUpdateCopy() throws SQLException { + Assert.assertFalse(metaData.locatorsUpdateCopy()); + } + + @Test + public void supportsStatementPooling() throws SQLException { + Assert.assertFalse(metaData.supportsStatementPooling()); + } + + @Test + public void getRowIdLifetime() throws SQLException { + Assert.assertNull(metaData.getRowIdLifetime()); + } + + @Test + public void supportsStoredFunctionsUsingCallSyntax() throws SQLException { + Assert.assertFalse(metaData.supportsStoredFunctionsUsingCallSyntax()); + } + + @Test + public void autoCommitFailureClosesAllResultSets() throws SQLException { + Assert.assertFalse(metaData.autoCommitFailureClosesAllResultSets()); + } + + @Test + public void getClientInfoProperties() throws SQLException { + Assert.assertNotNull(metaData.getClientInfoProperties()); + } + + @Test + public void getFunctions() throws SQLException { + Assert.assertNotNull(metaData.getFunctions("", "", "")); + } + + @Test + public void getFunctionColumns() throws SQLException { + Assert.assertNotNull(metaData.getFunctionColumns("", "", "", "")); + } + + @Test + public void getPseudoColumns() throws SQLException { + Assert.assertNotNull(metaData.getPseudoColumns("", "", "", "")); + } + + @Test + public void generatedKeyAlwaysReturned() throws SQLException { + Assert.assertFalse(metaData.generatedKeyAlwaysReturned()); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + connection = DriverManager.getConnection(url, properties); + metaData = connection.getMetaData().unwrap(RestfulDatabaseMetaData.class); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (connection != null) + connection.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index d07a6a21796fc2ea7927c04fd0cf5a37e9a49988..e239aa068bae0ee70d204aec4412bf29e5b36bf1 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.sql.*; public class RestfulDriverTest { + private static final String host = "127.0.0.1"; @Test public void connect() { @@ -15,9 +16,9 @@ public class RestfulDriverTest { @Test public void acceptsURL() throws SQLException { Driver driver = new RestfulDriver(); - boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://master:6041"); + boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://" + host + ":6041"); Assert.assertTrue(isAccept); - isAccept = driver.acceptsURL("jdbc:TAOS://master:6041"); + isAccept = driver.acceptsURL("jdbc:TAOS://" + host + ":6041"); Assert.assertFalse(isAccept); } @@ -26,6 +27,9 @@ public class RestfulDriverTest { Driver driver = new RestfulDriver(); final String url = ""; DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, null); + for (DriverPropertyInfo prop : propertyInfo) { + System.out.println(prop); + } } @Test diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java index 0af6b91532b63773510e10806d8f1e744ffe966e..c8bb69d82749e606f18d3298697ea0995029d064 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc.rs; - import org.junit.*; import org.junit.runners.MethodSorters; @@ -10,23 +9,12 @@ import java.util.Random; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class RestfulJDBCTest { - private Connection connection; - - @Before - public void before() throws ClassNotFoundException, SQLException { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/restful_test?user=root&password=taosdata"); - } - - @After - public void after() throws SQLException { - if (connection != null) - connection.close(); - } - + private static final String host = "127.0.0.1"; + private static Connection connection; + private Random random = new Random(System.currentTimeMillis()); /** - * 查询所有log.log + * select * from log.log **/ @Test public void testCase001() { @@ -85,7 +73,6 @@ public class RestfulJDBCTest { } } - private Random random = new Random(System.currentTimeMillis()); @Test public void testCase005() { @@ -105,5 +92,50 @@ public class RestfulJDBCTest { } } + @Test + public void testCase006() { + try (Statement stmt = connection.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + while (rs.next()) { + System.out.print("ts: " + rs.getTimestamp("ts")); + System.out.print(", temperature: " + rs.getString("temperature")); + System.out.print(", humidity: " + rs.getString("humidity")); + System.out.println(", location: " + rs.getString("location")); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase007() { + try (Statement stmt = connection.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + int columnCount = meta.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + String columnLabel = meta.getColumnLabel(i); + String value = rs.getString(i); + System.out.print(columnLabel + ": " + value + "\t"); + } + System.out.println(); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void before() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + } + + @AfterClass + public static void after() throws SQLException { + if (connection != null) + connection.close(); + } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..8bb2532ce82e24e4015746e04b5b2dea18f530e4 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulParameterMetaDataTest.java @@ -0,0 +1,194 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBConstants; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class RestfulParameterMetaDataTest { + + private static final String host = "127.0.0.1"; + private static Connection conn; + private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + private static PreparedStatement pstmt_insert; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; + private static PreparedStatement pstmt_select; + private static ParameterMetaData parameterMetaData_insert; + private static ParameterMetaData parameterMetaData_select; + + @Test + public void getParameterCount() throws SQLException { + Assert.assertEquals(10, parameterMetaData_insert.getParameterCount()); + } + + @Test + public void isNullable() throws SQLException { + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(1)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(2)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(3)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(4)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(5)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(6)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(7)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(8)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(9)); + Assert.assertEquals(ParameterMetaData.parameterNullableUnknown, parameterMetaData_insert.isNullable(10)); + } + + @Test + public void isSigned() throws SQLException { + Assert.assertEquals(false, parameterMetaData_insert.isSigned(1)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(2)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(3)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(4)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(5)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(6)); + Assert.assertEquals(true, parameterMetaData_insert.isSigned(7)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(8)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(9)); + Assert.assertEquals(false, parameterMetaData_insert.isSigned(10)); + } + + @Test + public void getPrecision() throws SQLException { + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(1)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(2)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(3)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(4)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(5)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(6)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(7)); + Assert.assertEquals(0, parameterMetaData_insert.getPrecision(8)); + Assert.assertEquals(5, parameterMetaData_insert.getPrecision(9)); + Assert.assertEquals(5, parameterMetaData_insert.getPrecision(10)); + } + + @Test + public void getScale() throws SQLException { + Assert.assertEquals(0, parameterMetaData_insert.getScale(1)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(2)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(3)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(4)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(5)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(6)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(7)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(8)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(9)); + Assert.assertEquals(0, parameterMetaData_insert.getScale(10)); + } + + @Test + public void getParameterType() throws SQLException { + Assert.assertEquals(Types.TIMESTAMP, parameterMetaData_insert.getParameterType(1)); + Assert.assertEquals(Types.INTEGER, parameterMetaData_insert.getParameterType(2)); + Assert.assertEquals(Types.BIGINT, parameterMetaData_insert.getParameterType(3)); + Assert.assertEquals(Types.FLOAT, parameterMetaData_insert.getParameterType(4)); + Assert.assertEquals(Types.DOUBLE, parameterMetaData_insert.getParameterType(5)); + Assert.assertEquals(Types.SMALLINT, parameterMetaData_insert.getParameterType(6)); + Assert.assertEquals(Types.TINYINT, parameterMetaData_insert.getParameterType(7)); + Assert.assertEquals(Types.BOOLEAN, parameterMetaData_insert.getParameterType(8)); + Assert.assertEquals(Types.BINARY, parameterMetaData_insert.getParameterType(9)); + Assert.assertEquals(Types.NCHAR, parameterMetaData_insert.getParameterType(10)); + } + + @Test + public void getParameterTypeName() throws SQLException { + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.TIMESTAMP), parameterMetaData_insert.getParameterTypeName(1)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.INTEGER), parameterMetaData_insert.getParameterTypeName(2)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BIGINT), parameterMetaData_insert.getParameterTypeName(3)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.FLOAT), parameterMetaData_insert.getParameterTypeName(4)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.DOUBLE), parameterMetaData_insert.getParameterTypeName(5)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.SMALLINT), parameterMetaData_insert.getParameterTypeName(6)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.TINYINT), parameterMetaData_insert.getParameterTypeName(7)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BOOLEAN), parameterMetaData_insert.getParameterTypeName(8)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.BINARY), parameterMetaData_insert.getParameterTypeName(9)); + Assert.assertEquals(TSDBConstants.jdbcType2TaosTypeName(Types.NCHAR), parameterMetaData_insert.getParameterTypeName(10)); + } + + @Test + public void getParameterClassName() throws SQLException { + Assert.assertEquals(Timestamp.class.getName(), parameterMetaData_insert.getParameterClassName(1)); + Assert.assertEquals(Integer.class.getName(), parameterMetaData_insert.getParameterClassName(2)); + Assert.assertEquals(Long.class.getName(), parameterMetaData_insert.getParameterClassName(3)); + Assert.assertEquals(Float.class.getName(), parameterMetaData_insert.getParameterClassName(4)); + Assert.assertEquals(Double.class.getName(), parameterMetaData_insert.getParameterClassName(5)); + Assert.assertEquals(Short.class.getName(), parameterMetaData_insert.getParameterClassName(6)); + Assert.assertEquals(Byte.class.getName(), parameterMetaData_insert.getParameterClassName(7)); + Assert.assertEquals(Boolean.class.getName(), parameterMetaData_insert.getParameterClassName(8)); + Assert.assertEquals(byte[].class.getName(), parameterMetaData_insert.getParameterClassName(9)); + Assert.assertEquals(String.class.getName(), parameterMetaData_insert.getParameterClassName(10)); + } + + @Test + public void getParameterMode() throws SQLException { + for (int i = 1; i <= parameterMetaData_insert.getParameterCount(); i++) { + int parameterMode = parameterMetaData_insert.getParameterMode(i); + Assert.assertEquals(ParameterMetaData.parameterModeUnknown, parameterMode); + } + } + + @Test + public void unwrap() throws SQLException { + RestfulParameterMetaData unwrap = parameterMetaData_insert.unwrap(RestfulParameterMetaData.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(parameterMetaData_insert.isWrapperFor(RestfulParameterMetaData.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_pstmt"); + stmt.execute("create database if not exists test_pstmt"); + stmt.execute("use test_pstmt"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + } + pstmt_insert = conn.prepareStatement(sql_insert); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(2, 111); + pstmt_insert.setObject(3, Long.MAX_VALUE); + pstmt_insert.setObject(4, 3.14159265354f); + pstmt_insert.setObject(5, Double.MAX_VALUE); + pstmt_insert.setObject(6, Short.MAX_VALUE); + pstmt_insert.setObject(7, Byte.MAX_VALUE); + pstmt_insert.setObject(8, true); + pstmt_insert.setObject(9, "hello".getBytes()); + pstmt_insert.setObject(10, "Hello"); + parameterMetaData_insert = pstmt_insert.getParameterMetaData(); + + pstmt_select = conn.prepareStatement(sql_select); + pstmt_select.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_select.setTimestamp(2, new Timestamp(System.currentTimeMillis() + 10000)); + pstmt_select.setInt(3, 0); + parameterMetaData_select = pstmt_select.getParameterMetaData(); + + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (pstmt_insert != null) + pstmt_insert.close(); + if (pstmt_select != null) + pstmt_select.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java new file mode 100644 index 0000000000000000000000000000000000000000..094dff8c8de0667110e4911a32a8479e21d6fc71 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulPreparedStatementTest.java @@ -0,0 +1,363 @@ +package com.taosdata.jdbc.rs; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.io.Serializable; +import java.sql.*; + +public class RestfulPreparedStatementTest { + private static final String host = "127.0.0.1"; + private static Connection conn; + private static final String sql_insert = "insert into t1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + private static PreparedStatement pstmt_insert; + private static final String sql_select = "select * from t1 where ts > ? and ts <= ? and f1 >= ?"; + private static PreparedStatement pstmt_select; + + @Test + public void executeQuery() throws SQLException { + long end = System.currentTimeMillis(); + long start = end - 1000 * 60 * 60; + pstmt_select.setTimestamp(1, new Timestamp(start)); + pstmt_select.setTimestamp(2, new Timestamp(end)); + pstmt_select.setInt(3, 0); + + ResultSet rs = pstmt_select.executeQuery(); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + } + + @Test + public void executeUpdate() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setFloat(4, 3.14f); + int result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + } + + @Test + public void setNull() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setNull(2, Types.INTEGER); + int result = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, result); + } + + @Test + public void setBoolean() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setBoolean(8, true); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setByte() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setByte(7, (byte) 0x001); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setShort() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setShort(6, (short) 2); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setInt() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setInt(2, 10086); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setLong() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setLong(3, Long.MAX_VALUE); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setFloat() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setFloat(4, 3.14f); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void setDouble() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setDouble(5, 3.14444); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBigDecimal() throws SQLException { + pstmt_insert.setBigDecimal(1, null); + } + + @Test + public void setString() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, "aaaa"); + boolean execute = pstmt_insert.execute(); + Assert.assertFalse(execute); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, new Person("john", 33, true).toString()); + Assert.assertFalse(pstmt_insert.execute()); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setString(10, new Person("john", 33, true).toString().replaceAll("'", "\"")); + Assert.assertFalse(pstmt_insert.execute()); + } + + class Person implements Serializable { + String name; + int age; + boolean sex; + + public Person(String name, int age, boolean sex) { + this.name = name; + this.age = age; + this.sex = sex; + } + + @Override + public String toString() { + return "Person{" + + "name='" + name + '\'' + + ", age=" + age + + ", sex=" + sex + + '}'; + } + } + + @Test + public void setBytes() throws SQLException, IOException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + +// ByteArrayOutputStream baos = new ByteArrayOutputStream(); +// ObjectOutputStream oos = new ObjectOutputStream(baos); +// oos.writeObject(new Person("john", 33, true)); +// oos.flush(); +// byte[] bytes = baos.toByteArray(); +// pstmt_insert.setBytes(9, bytes); + + pstmt_insert.setBytes(9, new Person("john", 33, true).toString().getBytes()); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setDate() throws SQLException { + pstmt_insert.setDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setTime() throws SQLException { + pstmt_insert.setTime(1, new Time(System.currentTimeMillis())); + } + + @Test + public void setTimestamp() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setAsciiStream() throws SQLException { + pstmt_insert.setAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBinaryStream() throws SQLException { + pstmt_insert.setBinaryStream(1, null); + } + + @Test + public void clearParameters() throws SQLException { + pstmt_insert.clearParameters(); + } + + @Test + public void setObject() throws SQLException { + pstmt_insert.setObject(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(2, 111); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(3, Long.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(4, 3.14159265354f); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(5, Double.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(6, Short.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(7, Byte.MAX_VALUE); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(8, true); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(9, "hello".getBytes()); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setObject(10, "Hello"); + ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + } + + @Test + public void execute() throws SQLException { + pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + int ret = pstmt_insert.executeUpdate(); + Assert.assertEquals(1, ret); + + executeQuery(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setCharacterStream() throws SQLException { + pstmt_insert.setCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setRef() throws SQLException { + pstmt_insert.setRef(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setBlob() throws SQLException { + pstmt_insert.setBlob(1, (Blob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setClob() throws SQLException { + pstmt_insert.setClob(1, (Clob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setArray() throws SQLException { + pstmt_insert.setArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getMetaData() throws SQLException { + pstmt_insert.getMetaData(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setURL() throws SQLException { + pstmt_insert.setURL(1, null); + } + + @Test + public void getParameterMetaData() throws SQLException { + ParameterMetaData parameterMetaData = pstmt_insert.getParameterMetaData(); + Assert.assertNotNull(parameterMetaData); + //TODO: modify the test case + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setRowId() throws SQLException { + pstmt_insert.setRowId(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNString() throws SQLException { + pstmt_insert.setNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNCharacterStream() throws SQLException { + pstmt_insert.setNCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setNClob() throws SQLException { + pstmt_insert.setNClob(1, (NClob) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setSQLXML() throws SQLException { + pstmt_insert.setSQLXML(1, null); + } + + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_pstmt"); + stmt.execute("create database if not exists test_pstmt"); + stmt.execute("use test_pstmt"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64)) tags(loc nchar(64))"); + stmt.execute("create table t1 using weather tags('beijing')"); + } + pstmt_insert = conn.prepareStatement(sql_insert); + pstmt_select = conn.prepareStatement(sql_select); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (pstmt_insert != null) + pstmt_insert.close(); + if (pstmt_select != null) + pstmt_select.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java new file mode 100644 index 0000000000000000000000000000000000000000..c7fc81297264f3cf38795d9d5a3b7eccc51574c9 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetMetaDataTest.java @@ -0,0 +1,220 @@ +package com.taosdata.jdbc.rs; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; + +public class RestfulResultSetMetaDataTest { + + private static final String host = "127.0.0.1"; + + private static Connection conn; + private static Statement stmt; + private static ResultSet rs; + private static ResultSetMetaData meta; + + @Test + public void getColumnCount() throws SQLException { + Assert.assertEquals(10, meta.getColumnCount()); + } + + @Test + public void isAutoIncrement() throws SQLException { + Assert.assertFalse(meta.isAutoIncrement(1)); + Assert.assertFalse(meta.isAutoIncrement(2)); + Assert.assertFalse(meta.isAutoIncrement(3)); + Assert.assertFalse(meta.isAutoIncrement(4)); + Assert.assertFalse(meta.isAutoIncrement(5)); + Assert.assertFalse(meta.isAutoIncrement(6)); + Assert.assertFalse(meta.isAutoIncrement(7)); + Assert.assertFalse(meta.isAutoIncrement(8)); + Assert.assertFalse(meta.isAutoIncrement(9)); + Assert.assertFalse(meta.isAutoIncrement(10)); + } + + @Test + public void isCaseSensitive() throws SQLException { + Assert.assertFalse(meta.isCaseSensitive(1)); + } + + @Test + public void isSearchable() throws SQLException { + Assert.assertTrue(meta.isSearchable(1)); + } + + @Test + public void isCurrency() throws SQLException { + Assert.assertFalse(meta.isCurrency(1)); + } + + @Test + public void isNullable() throws SQLException { + Assert.assertEquals(ResultSetMetaData.columnNoNulls, meta.isNullable(1)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(2)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(3)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(4)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(5)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(6)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(7)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(8)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(9)); + Assert.assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(10)); + } + + @Test + public void isSigned() throws SQLException { + Assert.assertFalse(meta.isSigned(1)); + } + + @Test + public void getColumnDisplaySize() throws SQLException { + Assert.assertEquals(64, meta.getColumnDisplaySize(10)); + } + + @Test + public void getColumnLabel() throws SQLException { + Assert.assertEquals("f1", meta.getColumnLabel(1)); + } + + @Test + public void getColumnName() throws SQLException { + Assert.assertEquals("f1", meta.getColumnName(1)); + } + + @Test + public void getSchemaName() throws SQLException { + Assert.assertEquals("", meta.getSchemaName(1)); + } + + @Test + public void getPrecision() throws SQLException { + Assert.assertEquals(0, meta.getPrecision(1)); + } + + @Test + public void getScale() throws SQLException { + Assert.assertEquals(0, meta.getScale(1)); + } + + @Test + public void getTableName() throws SQLException { + Assert.assertEquals("", meta.getTableName(1)); + } + + @Test + public void getCatalogName() throws SQLException { + Assert.assertEquals("restful_test", meta.getCatalogName(1)); + Assert.assertEquals("restful_test", meta.getCatalogName(2)); + Assert.assertEquals("restful_test", meta.getCatalogName(3)); + Assert.assertEquals("restful_test", meta.getCatalogName(4)); + Assert.assertEquals("restful_test", meta.getCatalogName(5)); + Assert.assertEquals("restful_test", meta.getCatalogName(6)); + Assert.assertEquals("restful_test", meta.getCatalogName(7)); + Assert.assertEquals("restful_test", meta.getCatalogName(8)); + Assert.assertEquals("restful_test", meta.getCatalogName(9)); + Assert.assertEquals("restful_test", meta.getCatalogName(10)); + } + + @Test + public void getColumnType() throws SQLException { + Assert.assertEquals(Types.TIMESTAMP, meta.getColumnType(1)); + Assert.assertEquals(Types.INTEGER, meta.getColumnType(2)); + Assert.assertEquals(Types.BIGINT, meta.getColumnType(3)); + Assert.assertEquals(Types.FLOAT, meta.getColumnType(4)); + Assert.assertEquals(Types.DOUBLE, meta.getColumnType(5)); + Assert.assertEquals(Types.BINARY, meta.getColumnType(6)); + Assert.assertEquals(Types.SMALLINT, meta.getColumnType(7)); + Assert.assertEquals(Types.TINYINT, meta.getColumnType(8)); + Assert.assertEquals(Types.BOOLEAN, meta.getColumnType(9)); + Assert.assertEquals(Types.NCHAR, meta.getColumnType(10)); + } + + @Test + public void getColumnTypeName() throws SQLException { + Assert.assertEquals("TIMESTAMP", meta.getColumnTypeName(1)); + Assert.assertEquals("INT", meta.getColumnTypeName(2)); + Assert.assertEquals("BIGINT", meta.getColumnTypeName(3)); + Assert.assertEquals("FLOAT", meta.getColumnTypeName(4)); + Assert.assertEquals("DOUBLE", meta.getColumnTypeName(5)); + Assert.assertEquals("BINARY", meta.getColumnTypeName(6)); + Assert.assertEquals("SMALLINT", meta.getColumnTypeName(7)); + Assert.assertEquals("TINYINT", meta.getColumnTypeName(8)); + Assert.assertEquals("BOOL", meta.getColumnTypeName(9)); + Assert.assertEquals("NCHAR", meta.getColumnTypeName(10)); + } + + @Test + public void isReadOnly() throws SQLException { + Assert.assertTrue(meta.isReadOnly(1)); + } + + @Test + public void isWritable() throws SQLException { + Assert.assertFalse(meta.isWritable(1)); + } + + @Test + public void isDefinitelyWritable() throws SQLException { + Assert.assertFalse(meta.isDefinitelyWritable(1)); + } + + @Test + public void getColumnClassName() throws SQLException { + Assert.assertEquals(Timestamp.class.getName(), meta.getColumnClassName(1)); + Assert.assertEquals(Integer.class.getName(), meta.getColumnClassName(2)); + Assert.assertEquals(Long.class.getName(), meta.getColumnClassName(3)); + Assert.assertEquals(Float.class.getName(), meta.getColumnClassName(4)); + Assert.assertEquals(Double.class.getName(), meta.getColumnClassName(5)); + Assert.assertEquals(String.class.getName(), meta.getColumnClassName(6)); + Assert.assertEquals(Short.class.getName(), meta.getColumnClassName(7)); + Assert.assertEquals(Short.class.getName(), meta.getColumnClassName(8)); + Assert.assertEquals(Boolean.class.getName(), meta.getColumnClassName(9)); + Assert.assertEquals(String.class.getName(), meta.getColumnClassName(10)); + } + + @Test + public void unwrap() throws SQLException { + Assert.assertNotNull(meta.unwrap(RestfulResultSetMetaData.class)); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(meta.isWrapperFor(RestfulResultSetMetaData.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + meta = rs.getMetaData(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (rs != null) + rs.close(); + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java new file mode 100644 index 0000000000000000000000000000000000000000..d6b2a58127d0440cf1c3a730439ccbeb94a77bdd --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulResultSetTest.java @@ -0,0 +1,692 @@ +package com.taosdata.jdbc.rs; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.google.common.primitives.Shorts; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.InputStream; +import java.io.Reader; +import java.math.BigDecimal; +import java.sql.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +public class RestfulResultSetTest { + + private static final String host = "127.0.0.1"; + + private static Connection conn; + private static Statement stmt; + private static ResultSet rs; + + @Test + public void wasNull() throws SQLException { + Assert.assertFalse(rs.wasNull()); + } + + @Test + public void getString() throws SQLException { + String f10 = rs.getString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test + public void getBoolean() throws SQLException { + Boolean f9 = rs.getBoolean("f9"); + Assert.assertEquals(true, f9); + f9 = rs.getBoolean(9); + Assert.assertEquals(true, f9); + } + + @Test + public void getByte() throws SQLException { + byte f8 = rs.getByte("f8"); + Assert.assertEquals(10, f8); + f8 = rs.getByte(8); + Assert.assertEquals(10, f8); + } + + @Test + public void getShort() throws SQLException { + short f7 = rs.getShort("f7"); + Assert.assertEquals(10, f7); + f7 = rs.getShort(7); + Assert.assertEquals(10, f7); + } + + @Test + public void getInt() throws SQLException { + int f2 = rs.getInt("f2"); + Assert.assertEquals(1, f2); + f2 = rs.getInt(2); + Assert.assertEquals(1, f2); + } + + @Test + public void getLong() throws SQLException { + long f3 = rs.getLong("f3"); + Assert.assertEquals(100, f3); + f3 = rs.getLong(3); + Assert.assertEquals(100, f3); + } + + @Test + public void getFloat() throws SQLException { + float f4 = rs.getFloat("f4"); + Assert.assertEquals(3.1415f, f4, 0f); + f4 = rs.getFloat(4); + Assert.assertEquals(3.1415f, f4, 0f); + } + + @Test + public void getDouble() throws SQLException { + double f5 = rs.getDouble("f5"); + Assert.assertEquals(3.1415926, f5, 0.0); + f5 = rs.getDouble(5); + Assert.assertEquals(3.1415926, f5, 0.0); + } + + @Test + public void getBigDecimal() throws SQLException { + BigDecimal f1 = rs.getBigDecimal("f1"); + Assert.assertEquals(1609430400000l, f1.longValue()); + + BigDecimal f2 = rs.getBigDecimal("f2"); + Assert.assertEquals(1, f2.intValue()); + + BigDecimal f3 = rs.getBigDecimal("f3"); + Assert.assertEquals(100l, f3.longValue()); + + BigDecimal f4 = rs.getBigDecimal("f4"); + Assert.assertEquals(3.1415f, f4.floatValue(), 0.00000f); + + BigDecimal f5 = rs.getBigDecimal("f5"); + Assert.assertEquals(3.1415926, f5.doubleValue(), 0.0000000); + + BigDecimal f7 = rs.getBigDecimal("f7"); + Assert.assertEquals(10, f7.intValue()); + + BigDecimal f8 = rs.getBigDecimal("f8"); + Assert.assertEquals(10, f8.intValue()); + } + + @Test + public void getBytes() throws SQLException { + byte[] f1 = rs.getBytes("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", new String(f1)); + + byte[] f2 = rs.getBytes("f2"); + Assert.assertEquals(1, Ints.fromByteArray(f2)); + + byte[] f3 = rs.getBytes("f3"); + Assert.assertEquals(100l, Longs.fromByteArray(f3)); + + byte[] f4 = rs.getBytes("f4"); + Assert.assertEquals(3.1415f, Float.valueOf(new String(f4)), 0.000000f); + + byte[] f5 = rs.getBytes("f5"); + Assert.assertEquals(3.1415926, Double.valueOf(new String(f5)), 0.000000f); + + byte[] f6 = rs.getBytes("f6"); + Assert.assertEquals("abc", new String(f6)); + + byte[] f7 = rs.getBytes("f7"); + Assert.assertEquals((short) 10, Shorts.fromByteArray(f7)); + + byte[] f8 = rs.getBytes("f8"); + Assert.assertEquals(1, f8.length); + Assert.assertEquals((byte) 10, f8[0]); + + byte[] f9 = rs.getBytes("f9"); + Assert.assertEquals("true", new String(f9)); + + byte[] f10 = rs.getBytes("f10"); + Assert.assertEquals("涛思数据", new String(f10)); + } + + @Test + public void getDate() throws SQLException, ParseException { + Date f1 = rs.getDate("f1"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Assert.assertEquals(sdf.parse("2021-01-01"), f1); + } + + @Test + public void getTime() throws SQLException { + Time f1 = rs.getTime("f1"); + Assert.assertEquals("00:00:00", f1.toString()); + } + + @Test + public void getTimestamp() throws SQLException { + Timestamp f1 = rs.getTimestamp("f1"); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + f1 = rs.getTimestamp(1); + Assert.assertEquals("2021-01-01 00:00:00.0", f1.toString()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getAsciiStream() throws SQLException { + rs.getAsciiStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getUnicodeStream() throws SQLException { + rs.getUnicodeStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBinaryStream() throws SQLException { + rs.getBinaryStream("f1"); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(rs.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + rs.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCursorName() throws SQLException { + rs.getCursorName(); + } + + @Test + public void getMetaData() throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertNotNull(meta); + } + + @Test + public void getObject() throws SQLException, ParseException { + Object f1 = rs.getObject("f1"); + Assert.assertEquals(Timestamp.class, f1.getClass()); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss.sss"); + java.util.Date date = sdf.parse("2021-01-01 00:00:00.000"); + Assert.assertEquals(new Timestamp(date.getTime()), f1); + + Object f2 = rs.getObject("f2"); + Assert.assertEquals(Integer.class, f2.getClass()); + Assert.assertEquals(1, f2); + + Object f3 = rs.getObject("f3"); + Assert.assertEquals(Long.class, f3.getClass()); + Assert.assertEquals(100l, f3); + + Object f4 = rs.getObject("f4"); + Assert.assertEquals(Float.class, f4.getClass()); + Assert.assertEquals(3.1415f, f4); + + Object f5 = rs.getObject("f5"); + Assert.assertEquals(Double.class, f5.getClass()); + Assert.assertEquals(3.1415926, f5); + + Object f6 = rs.getObject("f6"); + Assert.assertEquals(byte[].class, f6.getClass()); + Assert.assertEquals("abc", new String((byte[]) f6)); + + Object f7 = rs.getObject("f7"); + Assert.assertEquals(Short.class, f7.getClass()); + Assert.assertEquals((short) 10, f7); + + Object f8 = rs.getObject("f8"); + Assert.assertEquals(Byte.class, f8.getClass()); + Assert.assertEquals((byte) 10, f8); + + Object f9 = rs.getObject("f9"); + Assert.assertEquals(Boolean.class, f9.getClass()); + Assert.assertEquals(true, f9); + + Object f10 = rs.getObject("f10"); + Assert.assertEquals(String.class, f10.getClass()); + Assert.assertEquals("涛思数据", f10); + } + + @Test(expected = SQLException.class) + public void findColumn() throws SQLException { + int columnIndex = rs.findColumn("f1"); + Assert.assertEquals(1, columnIndex); + columnIndex = rs.findColumn("f2"); + Assert.assertEquals(2, columnIndex); + columnIndex = rs.findColumn("f3"); + Assert.assertEquals(3, columnIndex); + columnIndex = rs.findColumn("f4"); + Assert.assertEquals(4, columnIndex); + columnIndex = rs.findColumn("f5"); + Assert.assertEquals(5, columnIndex); + columnIndex = rs.findColumn("f6"); + Assert.assertEquals(6, columnIndex); + columnIndex = rs.findColumn("f7"); + Assert.assertEquals(7, columnIndex); + columnIndex = rs.findColumn("f8"); + Assert.assertEquals(8, columnIndex); + columnIndex = rs.findColumn("f9"); + Assert.assertEquals(9, columnIndex); + columnIndex = rs.findColumn("f10"); + Assert.assertEquals(10, columnIndex); + + rs.findColumn("f11"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getCharacterStream() throws SQLException { + rs.getCharacterStream(1); + } + + @Test + public void isBeforeFirst() throws SQLException { + Assert.assertFalse(rs.isBeforeFirst()); + rs.beforeFirst(); + Assert.assertTrue(rs.isBeforeFirst()); + rs.next(); + } + + @Test + public void isAfterLast() throws SQLException { + Assert.assertFalse(rs.isAfterLast()); + } + + @Test + public void isFirst() throws SQLException { + Assert.assertTrue(rs.isFirst()); + } + + @Test + public void isLast() throws SQLException { + Assert.assertTrue(rs.isLast()); + } + + @Test + public void beforeFirst() throws SQLException { + rs.beforeFirst(); + Assert.assertTrue(rs.isBeforeFirst()); + rs.next(); + } + + @Test + public void afterLast() throws SQLException { + rs.afterLast(); + Assert.assertTrue(rs.isAfterLast()); + rs.first(); + } + + @Test + public void first() throws SQLException { + rs.first(); + Assert.assertEquals("2021-01-01 00:00:00.0", rs.getTimestamp("f1").toString()); + } + + @Test + public void last() throws SQLException { + rs.last(); + Assert.assertEquals("2021-01-01 00:00:00.0", rs.getTimestamp("f1").toString()); + } + + @Test + public void getRow() throws SQLException { + int row = rs.getRow(); + Assert.assertEquals(1, row); + rs.beforeFirst(); + row = rs.getRow(); + Assert.assertEquals(0, row); + rs.first(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void absolute() throws SQLException { + rs.absolute(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void relative() throws SQLException { + rs.relative(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void previous() throws SQLException { + rs.previous(); + } + + @Test + public void setFetchDirection() throws SQLException { + rs.setFetchDirection(ResultSet.FETCH_FORWARD); + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + rs.setFetchDirection(ResultSet.FETCH_UNKNOWN); + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, rs.getFetchDirection()); + } + + @Test + public void setFetchSize() throws SQLException { + rs.setFetchSize(0); + Assert.assertEquals(0, rs.getFetchSize()); + } + + @Test + public void getFetchSize() throws SQLException { + Assert.assertEquals(0, rs.getFetchSize()); + } + + @Test + public void getType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, rs.getType()); + } + + @Test + public void getConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, rs.getConcurrency()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowUpdated() throws SQLException { + rs.rowUpdated(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowInserted() throws SQLException { + rs.rowInserted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void rowDeleted() throws SQLException { + rs.rowDeleted(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNull() throws SQLException { + rs.updateNull("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBoolean() throws SQLException { + rs.updateBoolean(1, false); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateByte() throws SQLException { + rs.updateByte(1, new Byte("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateShort() throws SQLException { + rs.updateShort(1, new Short("0")); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateInt() throws SQLException { + rs.updateInt(1, 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateLong() throws SQLException { + rs.updateLong(1, 1l); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateFloat() throws SQLException { + rs.updateFloat(1, 1f); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDouble() throws SQLException { + rs.updateDouble(1, 1.0); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBigDecimal() throws SQLException { + rs.updateBigDecimal(1, new BigDecimal(1)); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateString() throws SQLException { + rs.updateString(1, "abc"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBytes() throws SQLException { + rs.updateBytes(1, new byte[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateDate() throws SQLException { + rs.updateDate(1, new Date(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTime() throws SQLException { + rs.updateTime(1, new Time(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateTimestamp() throws SQLException { + rs.updateTimestamp(1, new Timestamp(System.currentTimeMillis())); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateAsciiStream() throws SQLException { + rs.updateAsciiStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBinaryStream() throws SQLException { + rs.updateBinaryStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateCharacterStream() throws SQLException { + rs.updateCharacterStream(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateObject() throws SQLException { + rs.updateObject(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void insertRow() throws SQLException { + rs.insertRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRow() throws SQLException { + rs.updateRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void deleteRow() throws SQLException { + rs.deleteRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void refreshRow() throws SQLException { + rs.refreshRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancelRowUpdates() throws SQLException { + rs.cancelRowUpdates(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToInsertRow() throws SQLException { + rs.moveToInsertRow(); + } + + @Test + public void getStatement() throws SQLException { + Statement stmt = rs.getStatement(); + Assert.assertNotNull(stmt); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void moveToCurrentRow() throws SQLException { + rs.moveToCurrentRow(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRef() throws SQLException { + rs.getRef(1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getBlob() throws SQLException { + rs.getBlob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getClob() throws SQLException { + rs.getClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getArray() throws SQLException { + rs.getArray("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getURL() throws SQLException { + rs.getURL("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRef() throws SQLException { + rs.updateRef("f1", null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateBlob() throws SQLException { + rs.updateBlob(1, (InputStream) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateClob() throws SQLException { + rs.updateClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateArray() throws SQLException { + rs.updateArray(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getRowId() throws SQLException { + rs.getRowId("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateRowId() throws SQLException { + rs.updateRowId(1, null); + } + + @Test + public void getHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, rs.getHoldability()); + } + + @Test + public void isClosed() throws SQLException { + Assert.assertFalse(rs.isClosed()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNString() throws SQLException { + rs.updateNString(1, null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNClob() throws SQLException { + rs.updateNClob(1, (Reader) null); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNClob() throws SQLException { + rs.getNClob("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getSQLXML() throws SQLException { + rs.getSQLXML("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateSQLXML() throws SQLException { + rs.updateSQLXML(1, null); + } + + @Test + public void getNString() throws SQLException { + String f10 = rs.getNString("f10"); + Assert.assertEquals("涛思数据", f10); + f10 = rs.getNString(10); + Assert.assertEquals("涛思数据", f10); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getNCharacterStream() throws SQLException { + rs.getNCharacterStream("f1"); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void updateNCharacterStream() throws SQLException { + rs.updateNCharacterStream(1, null); + } + + @Test + public void unwrap() throws SQLException { + RestfulResultSet unwrap = rs.unwrap(RestfulResultSet.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(rs.isWrapperFor(RestfulResultSet.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/restful_test?user=root&password=taosdata"); + stmt = conn.createStatement(); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table if not exists weather(f1 timestamp, f2 int, f3 bigint, f4 float, f5 double, f6 binary(64), f7 smallint, f8 tinyint, f9 bool, f10 nchar(64))"); + stmt.execute("insert into restful_test.weather values('2021-01-01 00:00:00.000', 1, 100, 3.1415, 3.1415926, 'abc', 10, 10, true, '涛思数据')"); + rs = stmt.executeQuery("select * from restful_test.weather"); + rs.next(); + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + + } + + @AfterClass + public static void afterClass() { + try { + if (rs != null) + rs.close(); + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java new file mode 100644 index 0000000000000000000000000000000000000000..1be32b502d3f8f7c1b94cd1a8940073520e11b12 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulStatementTest.java @@ -0,0 +1,417 @@ +package com.taosdata.jdbc.rs; + +import com.taosdata.jdbc.TSDBDriver; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.*; +import java.util.Properties; +import java.util.UUID; + +public class RestfulStatementTest { + private static final String host = "127.0.0.1"; + private static Connection conn; + private static Statement stmt; + + @Test + public void executeQuery() { + try { + ResultSet rs = stmt.executeQuery("show databases"); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + rs.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void executeUpdate() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + int affectRows = stmt.executeUpdate("create database " + dbName); + Assert.assertEquals(0, affectRows); + affectRows = stmt.executeUpdate("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(0, affectRows); + affectRows = stmt.executeUpdate("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(1, affectRows); + affectRows = stmt.executeUpdate("drop database " + dbName); + Assert.assertEquals(0, affectRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void close() { + // test in AfterClass method + } + + + @Test + public void getMaxFieldSize() throws SQLException { + Assert.assertEquals(16 * 1024, stmt.getMaxFieldSize()); + } + + @Test(expected = SQLException.class) + public void setMaxFieldSize() throws SQLException { + + stmt.setMaxFieldSize(0); + stmt.setMaxFieldSize(-1); + } + + @Test + public void getMaxRows() throws SQLException { + Assert.assertEquals(0, stmt.getMaxRows()); + } + + @Test(expected = SQLException.class) + public void setMaxRows() throws SQLException { + stmt.setMaxRows(0); + stmt.setMaxRows(-1); + } + + @Test + public void setEscapeProcessing() throws SQLException { + stmt.setEscapeProcessing(true); + stmt.setEscapeProcessing(false); + } + + @Test + public void getQueryTimeout() throws SQLException { + Assert.assertEquals(0, stmt.getQueryTimeout()); + } + + @Test(expected = SQLException.class) + public void setQueryTimeout() throws SQLException { + stmt.setQueryTimeout(0); + stmt.setQueryTimeout(-1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void cancel() throws SQLException { + stmt.cancel(); + } + + @Test + public void getWarnings() throws SQLException { + Assert.assertNull(stmt.getWarnings()); + } + + @Test + public void clearWarnings() throws SQLException { + stmt.clearWarnings(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void setCursorName() throws SQLException { + stmt.setCursorName(""); + } + + @Test + public void execute() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + boolean isSelect = stmt.execute("create database if not exists " + dbName); + Assert.assertEquals(false, isSelect); + int affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("create table if not exists " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(1, affectedRows); + + isSelect = stmt.execute("select * from " + dbName + ".weather"); + Assert.assertEquals(true, isSelect); + + isSelect = stmt.execute("drop database " + dbName); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getResultSet() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + boolean isSelect = stmt.execute("create database if not exists " + dbName); + Assert.assertEquals(false, isSelect); + int affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("create table if not exists " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + + isSelect = stmt.execute("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(1, affectedRows); + + isSelect = stmt.execute("select * from " + dbName + ".weather"); + Assert.assertEquals(true, isSelect); + ResultSet rs = stmt.getResultSet(); + Assert.assertNotNull(rs); + ResultSetMetaData meta = rs.getMetaData(); + Assert.assertEquals(3, meta.getColumnCount()); + int count = 0; + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + count++; + } + Assert.assertEquals(1, count); + + isSelect = stmt.execute("drop database " + dbName); + Assert.assertEquals(false, isSelect); + affectedRows = stmt.getUpdateCount(); + Assert.assertEquals(0, affectedRows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getUpdateCount() { + // already test in execute method + } + + @Test + public void getMoreResults() throws SQLException { + Assert.assertEquals(false, stmt.getMoreResults()); + } + + @Test(expected = SQLException.class) + public void setFetchDirection() throws SQLException { + stmt.setFetchDirection(ResultSet.FETCH_FORWARD); + stmt.setFetchDirection(ResultSet.FETCH_REVERSE); + stmt.setFetchDirection(ResultSet.FETCH_UNKNOWN); + stmt.setFetchDirection(-1); + } + + @Test + public void getFetchDirection() throws SQLException { + Assert.assertEquals(ResultSet.FETCH_FORWARD, stmt.getFetchDirection()); + } + + @Test(expected = SQLException.class) + public void setFetchSize() throws SQLException { + stmt.setFetchSize(0); + stmt.setFetchSize(-1); + } + + @Test + public void getFetchSize() throws SQLException { + stmt.setFetchSize(0); + Assert.assertEquals(0, stmt.getFetchSize()); + stmt.setFetchSize(0); + } + + @Test + public void getResultSetConcurrency() throws SQLException { + Assert.assertEquals(ResultSet.CONCUR_READ_ONLY, stmt.getResultSetConcurrency()); + } + + @Test + public void getResultSetType() throws SQLException { + Assert.assertEquals(ResultSet.TYPE_FORWARD_ONLY, stmt.getResultSetType()); + } + + @Test + public void addBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void clearBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.clearBatch(); + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + stmt.clearBatch(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void executeBatch() { + final String dbName = ("test_" + UUID.randomUUID()).replace("-", "_").substring(0, 32); + try { + stmt.addBatch("create database " + dbName); + stmt.addBatch("create table " + dbName + ".weather(ts timestamp, temperature float) tags(loc nchar(64))"); + stmt.addBatch("insert into " + dbName + ".t1 using " + dbName + ".weather tags('北京') values(now, 22.33)"); + stmt.addBatch("select * from " + dbName + ".weather"); + stmt.addBatch("drop database " + dbName); + int[] results = stmt.executeBatch(); + Assert.assertEquals(0, results[0]); + Assert.assertEquals(0, results[1]); + Assert.assertEquals(1, results[2]); + Assert.assertEquals(Statement.SUCCESS_NO_INFO, results[3]); + Assert.assertEquals(0, results[4]); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void getConnection() { + try { + Connection connection = stmt.getConnection(); + Assert.assertNotNull(connection); + Assert.assertTrue(this.conn == connection); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testGetMoreResults() throws SQLException { + Assert.assertEquals(false, stmt.getMoreResults(Statement.CLOSE_CURRENT_RESULT)); + stmt.getMoreResults(Statement.KEEP_CURRENT_RESULT); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getGeneratedKeys() throws SQLException { + stmt.getGeneratedKeys(); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate() throws SQLException { + stmt.executeUpdate("", 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate1() throws SQLException { + stmt.executeUpdate("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecuteUpdate2() throws SQLException { + stmt.executeUpdate("", new String[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute() throws SQLException { + stmt.execute("", 1); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute1() throws SQLException { + stmt.execute("", new int[]{}); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void testExecute2() throws SQLException { + stmt.execute("", new String[]{}); + } + + @Test + public void getResultSetHoldability() throws SQLException { + Assert.assertEquals(ResultSet.HOLD_CURSORS_OVER_COMMIT, stmt.getResultSetHoldability()); + } + + @Test + public void isClosed() { + try { + Assert.assertEquals(false, stmt.isClosed()); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void setPoolable() throws SQLException { + stmt.setPoolable(true); + stmt.setPoolable(false); + } + + @Test + public void isPoolable() throws SQLException { + Assert.assertEquals(false, stmt.isPoolable()); + } + + @Test + public void closeOnCompletion() throws SQLException { + stmt.closeOnCompletion(); + } + + @Test + public void isCloseOnCompletion() throws SQLException { + Assert.assertFalse(stmt.isCloseOnCompletion()); + } + + @Test + public void unwrap() throws SQLException { + RestfulStatement unwrap = stmt.unwrap(RestfulStatement.class); + Assert.assertNotNull(unwrap); + } + + @Test + public void isWrapperFor() throws SQLException { + Assert.assertTrue(stmt.isWrapperFor(RestfulStatement.class)); + } + + @BeforeClass + public static void beforeClass() { + try { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + conn = DriverManager.getConnection("jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata", properties); + stmt = conn.createStatement(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + if (stmt != null) + stmt.close(); + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java index 8ff308f8546b39519b9b24412e772a60a3ef68f0..4ad9826384a93e221b1181b72fa576bf72ebaff4 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/SQLTest.java @@ -1,5 +1,6 @@ package com.taosdata.jdbc.rs; +import com.taosdata.jdbc.utils.SQLExecutor; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -10,379 +11,327 @@ import java.sql.*; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class SQLTest { - private static final String host = "master"; - + private static final String host = "127.0.0.1"; private static Connection connection; @Test public void testCase001() { String sql = "create database if not exists restful_test"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase002() { String sql = "use restful_test"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase003() { String sql = "show databases"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase004() { String sql = "show tables"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase005() { String sql = "show stables"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase006() { String sql = "show dnodes"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase007() { String sql = "show vgroups"; - executeWithResult(sql); + SQLExecutor.executeWithResult(connection, sql); } @Test public void testCase008() { String sql = "drop table if exists restful_test.weather"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase009() { String sql = "create table if not exists restful_test.weather(ts timestamp, temperature float) tags(location nchar(64))"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase010() { String sql = "create table t1 using restful_test.weather tags('北京')"; - execute(sql); + SQLExecutor.execute(connection, sql); } @Test public void testCase011() { String sql = "insert into restful_test.t1 values(now, 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase012() { String sql = "insert into restful_test.t1 values('2020-01-01 00:00:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase013() { String sql = "insert into restful_test.t1 values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase014() { String sql = "insert into restful_test.t2 using weather tags('上海') values('2020-01-01 00:03:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase015() { String sql = "insert into restful_test.t2 using weather tags('上海') values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase016() { String sql = "insert into t1 values('2020-01-01 01:0:00.000', 22.22),('2020-01-01 02:00:00.000', 22.22) t2 values('2020-01-01 01:0:00.000', 33.33),('2020-01-01 02:00:00.000', 33.33)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase017() { String sql = "Insert into t3 using weather tags('广东') values('2020-01-01 01:0:00.000', 22.22),('2020-01-01 02:00:00.000', 22.22) t4 using weather tags('天津') values('2020-01-01 01:0:00.000', 33.33),('2020-01-01 02:00:00.000', 33.33)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase018() { String sql = "select * from restful_test.t1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase019() { String sql = "select * from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase020() { String sql = "select ts, temperature from restful_test.t1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase021() { String sql = "select ts, temperature from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase022() { String sql = "select temperature, ts from restful_test.t1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase023() { String sql = "select temperature, ts from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase024() { String sql = "import into restful_test.t5 using weather tags('石家庄') values('2020-01-01 00:01:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase025() { String sql = "import into restful_test.t6 using weather tags('沈阳') values('2020-01-01 00:01:00.000', 22.22),('2020-01-01 00:02:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase026() { String sql = "import into restful_test.t7 using weather tags('长沙') values('2020-01-01 00:01:00.000', 22.22) restful_test.t8 using weather tags('吉林') values('2020-01-01 00:01:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase027() { String sql = "import into restful_test.t9 using weather tags('武汉') values('2020-01-01 00:01:00.000', 22.22) ,('2020-01-02 00:01:00.000', 22.22) restful_test.t10 using weather tags('哈尔滨') values('2020-01-01 00:01:00.000', 22.22),('2020-01-02 00:01:00.000', 22.22)"; - executeUpdate(sql); + SQLExecutor.executeUpdate(connection, sql); } @Test public void testCase028() { String sql = "select location, temperature, ts from restful_test.weather where temperature > 1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase029() { String sql = "select location, temperature, ts from restful_test.weather where temperature < 1"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase030() { String sql = "select location, temperature, ts from restful_test.weather where ts > now"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase031() { String sql = "select location, temperature, ts from restful_test.weather where ts < now"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase032() { String sql = "select count(*) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase033() { String sql = "select first(*) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase034() { String sql = "select last(*) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase035() { String sql = "select last_row(*) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase036() { String sql = "select ts, ts as primary_key from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase037() { String sql = "select database()"; - execute("use restful_test"); - executeQuery(sql); + SQLExecutor.execute(connection, "use restful_test"); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase038() { String sql = "select client_version()"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase039() { String sql = "select server_status()"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase040() { String sql = "select server_status() as status"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase041() { String sql = "select tbname, location from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase042() { String sql = "select count(tbname) from restful_test.weather"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase043() { String sql = "select * from restful_test.weather where ts < now - 1h"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase044() { String sql = "select * from restful_test.weather where ts < now - 1h and location like '%'"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase045() { String sql = "select * from restful_test.weather where ts < now - 1h order by ts"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase046() { String sql = "select last(*) from restful_test.weather where ts < now - 1h group by tbname order by tbname"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase047() { String sql = "select * from restful_test.weather limit 2"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase048() { String sql = "select * from restful_test.weather limit 2 offset 5"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase049() { String sql = "select * from restful_test.t1, restful_test.t3 where t1.ts = t3.ts "; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase050() { String sql = "select * from restful_test.t1, restful_test.t3 where t1.ts = t3.ts and t1.location = t3.location"; - executeQuery(sql); + SQLExecutor.executeQuery(connection, sql); } @Test public void testCase051() { String sql = "select * from restful_test.t1 tt, restful_test.t3 yy where tt.ts = yy.ts"; - executeQuery(sql); - } - - private void executeUpdate(String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - int affectedRows = statement.executeUpdate(sql); - long end = System.currentTimeMillis(); - System.out.println("[ affected rows : " + affectedRows + " ] time cost: " + (end - start) + " ms, execute statement ====> " + sql); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - private void executeWithResult(String sql) { - try (Statement statement = connection.createStatement()) { - statement.execute(sql); - ResultSet resultSet = statement.getResultSet(); - printResult(resultSet); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - private void execute(String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - boolean execute = statement.execute(sql); - long end = System.currentTimeMillis(); - printSql(sql, execute, (end - start)); - } catch (SQLException e) { - System.out.println("ERROR execute SQL ===> " + sql); - e.printStackTrace(); - } - } - - private static void printSql(String sql, boolean succeed, long cost) { - System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); - } - - private void executeQuery(String sql) { - try (Statement statement = connection.createStatement()) { - long start = System.currentTimeMillis(); - ResultSet resultSet = statement.executeQuery(sql); - long end = System.currentTimeMillis(); - printSql(sql, true, (end - start)); - printResult(resultSet); - } catch (SQLException e) { - System.out.println("ERROR execute SQL ===> " + sql); - e.printStackTrace(); - } - } - - private static void printResult(ResultSet resultSet) throws SQLException { - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - StringBuilder sb = new StringBuilder(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String columnLabel = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - sb.append(columnLabel + ": " + value + "\t"); - } - System.out.println(sb.toString()); - } + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase052() { + String sql = "select server_status()"; + SQLExecutor.executeQuery(connection, sql); + } + + @Test + public void testCase053() { + String sql = "select avg(cpu_taosd), avg(cpu_system), max(cpu_cores), avg(mem_taosd), avg(mem_system), max(mem_total), avg(disk_used), max(disk_total), avg(band_speed), avg(io_read), avg(io_write), sum(req_http), sum(req_select), sum(req_insert) from log.dn1 where ts> now - 60m and ts<= now interval(1m) fill(value, 0)"; + SQLExecutor.executeQuery(connection, sql); } @BeforeClass diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java new file mode 100644 index 0000000000000000000000000000000000000000..fd6c83ad1cacfb351bcc1d35a5cf777213ad876d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java @@ -0,0 +1,30 @@ +package com.taosdata.jdbc.utils; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class OSUtilsTest { + + private String OS; + + @Test + public void inWindows() { + Assert.assertEquals(OS.indexOf("win") >= 0, OSUtils.isWindows()); + } + + @Test + public void isMac() { + Assert.assertEquals(OS.indexOf("mac") >= 0, OSUtils.isMac()); + } + + @Test + public void isLinux() { + Assert.assertEquals(OS.indexOf("nux") >= 0, OSUtils.isLinux()); + } + + @Before + public void before() { + OS = System.getProperty("os.name").toLowerCase(); + } +} \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java new file mode 100644 index 0000000000000000000000000000000000000000..bf034bf458bcb8eadaaacb5cf633f0905a8c1bd6 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SQLExecutor.java @@ -0,0 +1,74 @@ +package com.taosdata.jdbc.utils; + +import java.sql.*; + +public class SQLExecutor { + + // insert, import + public static void executeUpdate(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + int affectedRows = statement.executeUpdate(sql); + long end = System.currentTimeMillis(); + System.out.println("[ affected rows : " + affectedRows + " ] time cost: " + (end - start) + " ms, execute statement ====> " + sql); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // show databases, show tables, show stables + public static void executeWithResult(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + ResultSet resultSet = statement.getResultSet(); + printResult(resultSet); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + // use database, create database, create table, drop table... + public static void execute(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + boolean execute = statement.execute(sql); + long end = System.currentTimeMillis(); + printSql(sql, execute, (end - start)); + } catch (SQLException e) { + System.out.println("ERROR execute SQL ===> " + sql); + e.printStackTrace(); + } + } + + // select + public static void executeQuery(Connection connection, String sql) { + try (Statement statement = connection.createStatement()) { + long start = System.currentTimeMillis(); + ResultSet resultSet = statement.executeQuery(sql); + long end = System.currentTimeMillis(); + printSql(sql, true, (end - start)); + printResult(resultSet); + } catch (SQLException e) { + System.out.println("ERROR execute SQL ===> " + sql); + e.printStackTrace(); + } + } + + private static void printSql(String sql, boolean succeed, long cost) { + System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private static void printResult(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder sb = new StringBuilder(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + sb.append(columnLabel + ": " + value + "\t"); + } + System.out.println(sb.toString()); + } + } + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java index ce84f967d09e2217bac90cc7a3f59bf4f50b2a15..ccb0941da00e2755eaf8fde2553b8a8e6b33cd25 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -16,9 +16,6 @@ public class SqlSyntaxValidatorTest { @Test public void isUseSQL() { Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database if not exist test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database test")); - Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database if exist test")); } + } \ No newline at end of file diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..16f8269d246f094558bbe7f6b11ac4bd90eb888f --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/TimestampUtil.java @@ -0,0 +1,67 @@ +package com.taosdata.jdbc.utils; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +public class TimestampUtil { + + private static final String datetimeFormat = "yyyy-MM-dd HH:mm:ss.SSS"; + + public static long datetimeToLong(String dateTime) { + SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); + try { + return sdf.parse(dateTime).getTime(); + } catch (ParseException e) { + throw new IllegalArgumentException("invalid datetime string >>> " + dateTime); + } + } + + public static String longToDatetime(long time) { + SimpleDateFormat sdf = new SimpleDateFormat(datetimeFormat); + return sdf.format(new Date(time)); + } + + public static class TimeTuple { + public Long start; + public Long end; + public Long timeGap; + + TimeTuple(long start, long end, long timeGap) { + this.start = start; + this.end = end; + this.timeGap = timeGap; + } + } + + public static TimeTuple range(long start, long timeGap, long size) { + long now = System.currentTimeMillis(); + if (timeGap < 1) + timeGap = 1; + if (start == 0) + start = now - size * timeGap; + + // 如果size小于1异常 + if (size < 1) + throw new IllegalArgumentException("size less than 1."); + // 如果timeGap为1,已经超长,需要前移start + if (start + size > now) { + start = now - size; + return new TimeTuple(start, now, 1); + } + long end = start + (long) (timeGap * size); + if (end > now) { + //压缩timeGap + end = now; + double gap = (end - start) / (size * 1.0f); + if (gap < 1.0f) { + timeGap = 1; + start = end - size; + } else { + timeGap = (long) gap; + end = start + (long) (timeGap * size); + } + } + return new TimeTuple(start, end, timeGap); + } +} diff --git a/src/connector/nodejs/nodetaos/cinterface.js b/src/connector/nodejs/nodetaos/cinterface.js index a246256f15f313b61c0b7e990e63848fee2431bf..43a08a800afe7aa3e445a943f8d7a7f95f1ed05f 100644 --- a/src/connector/nodejs/nodetaos/cinterface.js +++ b/src/connector/nodejs/nodetaos/cinterface.js @@ -3,15 +3,15 @@ * @module CTaosInterface */ -const ref = require('ref'); +const ref = require('ref-napi'); const os = require('os'); -const ffi = require('ffi'); -const ArrayType = require('ref-array'); -const Struct = require('ref-struct'); +const ffi = require('ffi-napi'); +const ArrayType = require('ref-array-napi'); +const Struct = require('ref-struct-napi'); const FieldTypes = require('./constants'); const errors = require ('./error'); const TaosObjects = require('./taosobjects'); -const { NULL_POINTER } = require('ref'); +const { NULL_POINTER } = require('ref-napi'); module.exports = CTaosInterface; diff --git a/src/connector/nodejs/nodetaos/cursor.js b/src/connector/nodejs/nodetaos/cursor.js index 0c9214fe32656fdfb7edb6cfc0d0c3587d4f7bb6..e18e6c2500019684dea85fe05bc7306e0a372478 100644 --- a/src/connector/nodejs/nodetaos/cursor.js +++ b/src/connector/nodejs/nodetaos/cursor.js @@ -1,4 +1,4 @@ -const ref = require('ref'); +const ref = require('ref-napi'); require('./globalfunc.js') const CTaosInterface = require('./cinterface') const errors = require ('./error') diff --git a/src/connector/nodejs/package-lock.json b/src/connector/nodejs/package-lock.json index d13fe6959c07955f5b15c7281f571ab932a996c3..9ca174ccd12f782b047789e44a6abce231b15ad0 100644 --- a/src/connector/nodejs/package-lock.json +++ b/src/connector/nodejs/package-lock.json @@ -1,44 +1,9 @@ { "name": "td2.0-connector", - "version": "2.0.1", + "version": "2.0.6", "lockfileVersion": 1, "requires": true, "dependencies": { - "abbrev": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" - }, - "ajv": { - "version": "6.10.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", - "integrity": "sha512-TXtUUEYHuaTEbLZWIKUr5pmBuhDLy+8KYtPYdcV8qC+pOZL+NKqYwvWSRrVXHn+ZmRRAu8vJTAznH7Oag6RVRw==", - "requires": { - "fast-deep-equal": "^2.0.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, - "aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" - }, - "are-we-there-yet": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", - "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, "array-index": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/array-index/-/array-index-1.0.0.tgz", @@ -46,104 +11,23 @@ "requires": { "debug": "^2.2.0", "es6-symbol": "^3.0.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } } }, - "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "requires": { - "safer-buffer": "~2.1.0" - } - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" - }, - "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "requires": { - "tweetnacl": "^0.14.3" - } - }, - "bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "requires": { - "file-uri-to-path": "1.0.0" - } - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "chownr": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz", - "integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A==" - }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" - }, - "combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "d": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", @@ -153,54 +37,22 @@ "type": "^1.0.1" } }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "requires": { - "assert-plus": "^1.0.0" - } - }, "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, - "delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" - }, - "ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", "requires": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" + "ms": "2.1.2" } }, - "env-paths": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-1.0.0.tgz", - "integrity": "sha1-QWgTO0K7BcOKNbGuQ5fIKYqzaeA=" - }, "es5-ext": { - "version": "0.10.50", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.50.tgz", - "integrity": "sha512-KMzZTPBkeQV/JcSQhI5/z6d9VWJ3EnQ194USTUwIYZ2ZbpN8+SGXQKt1h68EX44+qt+Fzr8DO17vnxrw7c3agw==", + "version": "0.10.53", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz", + "integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==", "requires": { "es6-iterator": "~2.0.3", - "es6-symbol": "~3.1.1", - "next-tick": "^1.0.0" + "es6-symbol": "~3.1.3", + "next-tick": "~1.0.0" } }, "es6-iterator": { @@ -214,631 +66,220 @@ } }, "es6-symbol": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz", - "integrity": "sha1-vwDvT9q2uhtG7Le2KbTH7VcVzHc=", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", + "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", "requires": { - "d": "1", - "es5-ext": "~0.10.14" + "d": "^1.0.1", + "ext": "^1.1.2" } }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" - }, - "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" - }, - "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" - }, - "ffi": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/ffi/-/ffi-2.3.0.tgz", - "integrity": "sha512-vkPA9Hf9CVuQ5HeMZykYvrZF2QNJ/iKGLkyDkisBnoOOFeFXZQhUPxBARPBIZMJVulvBI2R+jgofW03gyPpJcQ==", + "ext": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz", + "integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==", "requires": { - "bindings": "~1.2.0", - "debug": "2", - "nan": "2", - "ref": "1", - "ref-struct": "1" + "type": "^2.0.0" }, "dependencies": { - "bindings": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", - "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE=" + "type": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz", + "integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA==" } } }, - "file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" - }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" - }, - "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, - "fs-minipass": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.6.tgz", - "integrity": "sha512-crhvyXcMejjv3Z5d2Fa9sf5xLYVCF5O1c71QxbVnbLsmYMBEvDAftewesN/HhY03YRoA7zOMxjNGrF5svGaaeQ==", - "requires": { - "minipass": "^2.2.1" - } - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "graceful-fs": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.0.tgz", - "integrity": "sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg==" - }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" - }, - "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "ffi-napi": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-3.1.0.tgz", + "integrity": "sha512-EsHO+sP2p/nUC/3l/l8m9niee1BLm4asUFDzkkBGR4kYVgp2KqdAYUomZhkKtzim4Fq7mcYHjpUaIHsMqs+E1g==", "requires": { - "ajv": "^6.5.5", - "har-schema": "^2.0.0" + "debug": "^4.1.1", + "get-uv-event-loop-napi-h": "^1.0.5", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1", + "ref-napi": "^2.0.1", + "ref-struct-di": "^1.1.0" + }, + "dependencies": { + "ref-napi": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-2.1.2.tgz", + "integrity": "sha512-aFl+vrIuLWUXMUTQGAwGAuSNLX3Ub5W3iVP8b7KyFFZUdn4+i4U1TXXTop0kCTUfGNu8glBGVz4lowkwMcPVVA==", + "requires": { + "debug": "^4.1.1", + "get-symbol-from-current-process-h": "^1.0.2", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1" + } + } } }, - "has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" - }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "requires": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - } + "get-symbol-from-current-process-h": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz", + "integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw==" }, - "inflight": { + "get-uv-event-loop-napi-h": { "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz", + "integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==", "requires": { - "number-is-nan": "^1.0.0" - } - }, - "is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" - }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, - "mime-db": { - "version": "1.40.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", - "integrity": "sha512-jYdeOMPy9vnxEqFRRo6ZvTZ8d9oPb+k18PKoYNYUe2stVEBPPwsln/qWzdbmaIvnhZ9v2P+CuecK+fpUfsV2mA==" - }, - "mime-types": { - "version": "2.1.24", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.24.tgz", - "integrity": "sha512-WaFHS3MCl5fapm3oLxU4eYDw77IQM2ACcxQ9RIxfaC3ooc6PFuBMGZZsYpvoXS5D5QTWPieo1jjLdAm3TBP3cQ==", - "requires": { - "mime-db": "1.40.0" - } - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - }, - "minipass": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", - "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", - "requires": { - "safe-buffer": "^5.1.2", - "yallist": "^3.0.0" - } - }, - "minizlib": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", - "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", - "requires": { - "minipass": "^2.2.1" - } - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" + "get-symbol-from-current-process-h": "^1.0.1" } }, "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "nan": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", - "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==" + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "next-tick": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz", "integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw=" }, - "node-gyp": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-5.0.2.tgz", - "integrity": "sha512-sNcb5O7eJ9XiNAhWZ/UE2bWsBJn3Jb7rayMqMP4wjenlr1DwzZxUmbtmIrl04EU0p5fN2rU9WIDV+u0EbsI8oQ==", - "requires": { - "env-paths": "^1.0.0", - "glob": "^7.0.3", - "graceful-fs": "^4.1.2", - "mkdirp": "^0.5.0", - "nopt": "2 || 3", - "npmlog": "0 || 1 || 2 || 3 || 4", - "request": "^2.87.0", - "rimraf": "2", - "semver": "~5.3.0", - "tar": "^4.4.8", - "which": "1" - } - }, - "nopt": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", - "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=", - "requires": { - "abbrev": "1" - } - }, - "npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" - }, - "oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" - }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" - } - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" - }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "psl": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.2.0.tgz", - "integrity": "sha512-GEn74ZffufCmkDDLNcl3uuyF/aSD6exEyh1v/ZSdAomB82t6G9hzJVRx0jBmLDW+VfZqks3aScmMw9DszwUalA==" - }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "node-addon-api": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz", + "integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA==" }, - "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" + "node-gyp-build": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", + "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==" }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "ref": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ref/-/ref-1.3.5.tgz", - "integrity": "sha512-2cBCniTtxcGUjDpvFfVpw323a83/0RLSGJJY5l5lcomZWhYpU2cuLdsvYqMixvsdLJ9+sTdzEkju8J8ZHDM2nA==", - "requires": { - "bindings": "1", - "debug": "2", - "nan": "2" - } - }, - "ref-array": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/ref-array/-/ref-array-1.2.0.tgz", - "integrity": "sha1-u6hwFS1O4KvtFCGwYjkvCwGEKQw=", + "ref-array-napi": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ref-array-napi/-/ref-array-napi-1.2.1.tgz", + "integrity": "sha512-jQp2WWSucmxkqVfoNfm7yDlDeGu3liAbzqfwjNybL80ooLOCnCZpAK2woDInY+lxNOK/VlIVSqeDEYb4gVPuNQ==", "requires": { "array-index": "1", "debug": "2", - "ref": "1" - } - }, - "ref-struct": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/ref-struct/-/ref-struct-1.1.0.tgz", - "integrity": "sha1-XV7mWtQc78Olxf60BYcmHkee3BM=", - "requires": { - "debug": "2", - "ref": "1" - } - }, - "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.0", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - } - }, - "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "requires": { - "glob": "^7.1.3" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "semver": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", - "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=" - }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, - "signal-exit": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" - }, - "sshpk": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", - "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", - "requires": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" + "ref-napi": "^1.4.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "ref-napi": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", + "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", + "requires": { + "debug": "^3.1.0", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + } } }, - "strip-ansi": { + "ref-napi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "tar": { - "version": "4.4.10", - "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.10.tgz", - "integrity": "sha512-g2SVs5QIxvo6OLp0GudTqEf05maawKUxXru104iaayWA09551tFCTI8f1Asb4lPfkBr91k07iL4c11XO3/b0tA==", + "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.1.tgz", + "integrity": "sha512-W3rcb0E+tlO9u9ySFnX5vifInwwPGToOfFgTZUHJBNiOBsW0NNvgHz2zJN7ctABo/2yIlgdPQUvuqqfORIF4LA==", "requires": { - "chownr": "^1.1.1", - "fs-minipass": "^1.2.5", - "minipass": "^2.3.5", - "minizlib": "^1.2.1", - "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.2", - "yallist": "^3.0.3" + "debug": "^4.1.1", + "get-symbol-from-current-process-h": "^1.0.2", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1" } }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "ref-struct-di": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz", + "integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==", "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" + "debug": "^3.1.0" }, "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } } } }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "ref-struct-napi": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ref-struct-napi/-/ref-struct-napi-1.1.1.tgz", + "integrity": "sha512-YgS5/d7+kT5zgtySYI5ieH0hREdv+DabgDvoczxsui0f9VLm0rrDcWEj4DHKehsH+tJnVMsLwuyctWgvdEcVRw==", "requires": { - "safe-buffer": "^5.0.1" + "debug": "2", + "ref-napi": "^1.4.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + }, + "ref-napi": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz", + "integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==", + "requires": { + "debug": "^3.1.0", + "node-addon-api": "^2.0.0", + "node-gyp-build": "^4.2.1" + }, + "dependencies": { + "debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + } } }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, "type": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/type/-/type-1.0.1.tgz", - "integrity": "sha512-MAM5dBMJCJNKs9E7JXo4CXRAansRfG0nlJxW7Wf6GZzSOvH31zClSaHdIMWLehe/EGMBkqeC55rrkaOr5Oo7Nw==" - }, - "uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "requires": { - "punycode": "^2.1.0" - } - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "requires": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - }, - "which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "requires": { - "isexe": "^2.0.0" - } - }, - "wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "requires": { - "string-width": "^1.0.2 || 2" - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" } } } diff --git a/src/connector/nodejs/package.json b/src/connector/nodejs/package.json index 5587a69e01e6743a7eb7ef234d111c86db246841..b39ce2c17d8e693d0e981885aed55ba1b82313b6 100644 --- a/src/connector/nodejs/package.json +++ b/src/connector/nodejs/package.json @@ -1,31 +1,34 @@ { "name": "td2.0-connector", - "version": "2.0.5", + "version": "2.0.6", "description": "A Node.js connector for TDengine.", "main": "tdengine.js", + "directories": { + "test": "test" + }, "scripts": { "test": "node test/test.js" }, + "repository": { + "type": "git", + "url": "git+https://github.com/taosdata/tdengine.git" + }, "keywords": [ "TDengine", "TAOS Data", "Time Series Database", "Connector" ], - "author": "StoneT2000", - "license": "AGPL-3.0", - "dependencies": { - "ffi": "^2.3.0", - "node-gyp": "^5.0.2", - "ref": "^1.3.5", - "ref-array": "^1.2.0" - }, - "homepage": "https://github.com/taosdata/tdengine", + "author": "TaosData Inc.", + "license": "AGPL-3.0-or-later", "bugs": { "url": "https://github.com/taosdata/tdengine/issues" }, - "repository": { - "type": "git", - "url": "https://github.com/taosdata/tdengine.git" + "homepage": "https://github.com/taosdata/tdengine#readme", + "dependencies": { + "ffi-napi": "^3.1.0", + "ref-array-napi": "^1.2.1", + "ref-napi": "^1.5.2", + "ref-struct-napi": "^1.1.1" } } diff --git a/src/connector/odbc/.gitignore b/src/connector/odbc/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..bfef9b2efaaf8cf6071c42687a1c4c02688a9eee --- /dev/null +++ b/src/connector/odbc/.gitignore @@ -0,0 +1,3 @@ +!c/ +node_modules/ +package-lock.json diff --git a/src/connector/odbc/CMakeLists.txt b/src/connector/odbc/CMakeLists.txt index 2e1e39ef123c901b36050a9e239fe58d26869e8e..5a93ac3f7e2934fd8383c5a18f22c24845793f1a 100644 --- a/src/connector/odbc/CMakeLists.txt +++ b/src/connector/odbc/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_LINUX_64) @@ -15,7 +15,7 @@ IF (TD_LINUX_64) message(STATUS "unixodbc/unixodbc-dev are installed, and odbc connector will be built") find_package(FLEX) if(NOT FLEX_FOUND) - message(FATAL_ERROR "you need to install flex first") + message(WARNING "you need to install flex first") else () if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0.0) message(WARNING "gcc 4.8.0 will complain too much about flex-generated code, we just bypass building ODBC driver in such case") @@ -24,7 +24,7 @@ IF (TD_LINUX_64) SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Wconversion") ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(tools) - ADD_SUBDIRECTORY(tests) + ADD_SUBDIRECTORY(examples) endif () endif() endif() @@ -33,10 +33,44 @@ IF (TD_LINUX_64) ENDIF () ENDIF () +IF (TD_DARWIN) + find_program(HAVE_ODBCINST NAMES odbcinst) + IF (HAVE_ODBCINST) + include(CheckSymbolExists) + # shall we revert CMAKE_REQUIRED_LIBRARIES and how? + set(CMAKE_REQUIRED_LIBRARIES odbc) + set(CMAKE_REQUIRED_INCLUDES /usr/local/include) + set(CMAKE_REQUIRED_LINK_OPTIONS -L/usr/local/lib) + check_symbol_exists(SQLExecute "sql.h" HAVE_ODBC_DEV) + if(NOT (HAVE_ODBC_DEV)) + unset(HAVE_ODBC_DEV CACHE) + message(WARNING "unixodbc-dev is not installed yet, you may install it with homebrew by typing: brew install unixodbc") + else () + message(STATUS "unixodbc/unixodbc-dev are installed, and odbc connector will be built") + find_package(FLEX) + if(NOT FLEX_FOUND) + message(WARNING "you need to install flex first") + else () + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0.0) + message(WARNING "gcc 4.8.0 will complain too much about flex-generated code, we just bypass building ODBC driver in such case") + else () + SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -Wconversion") + SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Wconversion") + ADD_SUBDIRECTORY(src) + ADD_SUBDIRECTORY(tools) + ADD_SUBDIRECTORY(examples) + endif () + endif() + endif() + ELSE () + message(WARNING "unixodbc is not installed yet, you may install it under ubuntu by typing: brew install unixodbc") + ENDIF () +ENDIF () + IF (TD_WINDOWS_64) find_package(ODBC) if (NOT ODBC_FOUND) - message(FATAL_ERROR "you need to install ODBC first") + message(WARNING "you need to install ODBC first") else () message(STATUS "ODBC_INCLUDE_DIRS: ${ODBC_INCLUDE_DIRS}") message(STATUS "ODBC_LIBRARIES: ${ODBC_LIBRARIES}") @@ -50,6 +84,7 @@ IF (TD_WINDOWS_64) else () ADD_SUBDIRECTORY(src) ADD_SUBDIRECTORY(tools) - ADD_SUBDIRECTORY(tests) + ADD_SUBDIRECTORY(examples) endif() ENDIF () + diff --git a/src/connector/odbc/README.cn.md b/src/connector/odbc/README.cn.md new file mode 100644 index 0000000000000000000000000000000000000000..bf114356a6329a6299599734e81b2cafc8769132 --- /dev/null +++ b/src/connector/odbc/README.cn.md @@ -0,0 +1,169 @@ + +# ODBC 驱动 # + +- **TAOS ODBC驱动持续更新中** + +- **目前导出的ODBC函数包括(注: 有些不常用的函数只是导出,但并未实现)**: +SQLAllocEnv +SQLFreeEnv +SQLAllocConnect +SQLFreeConnect +SQLGetEnvAttr +SQLSetEnvAttr +SQLGetConnectAttr +SQLGetConnectOption +SQLGetInfo +SQLConnect +SQLDisconnect +SQLAllocStmt +SQLAllocHandle +SQLFreeHandle +SQLFreeStmt +SQLExecDirect +SQLNumResultCols +SQLRowCount +SQLColAttribute +SQLGetData +SQLFetch +SQLPrepare +SQLExecute +SQLParamData +SQLPutData +SQLGetDiagRec +SQLBindParameter +SQLDescribeParam +SQLDriverConnect +SQLSetConnectAttr +SQLDescribeCol +SQLBindCol +SQLNumParams +SQLSetStmtAttr +SQLBindParam +SQLCancel +SQLCancelHandle +SQLCloseCursor +SQLColumns +SQLCopyDesc +SQLDataSources +SQLEndTran +SQLFetchScroll +SQLGetCursorName +SQLGetDescField +SQLGetDescRec +SQLGetStmtAttr +SQLGetStmtOption +SQLGetTypeInfo +SQLSetConnectOption +SQLSetCursorName +SQLSetDescField +SQLSetDescRec +SQLSetParam +SQLSetStmtOption +SQLSpecialColumns +SQLStatistics +SQLTables +SQLTransact + +` + +- **国际化。可以通过在ODBC连接串中指定针对SQLCHAR/SQLWCHAR/taos_charset/system-locale的字符集来解决常见的环境匹配问题**. + +- **现有的ODBC客户端工具可以籍此驱动同TAOS数据源互联,包括主流linux/macosx/windows平台** + +- **现有的支持ODBC的编程语言可以籍此驱动同TAOS数据源互联, 例如: c/nodejs/python/rust/go已经在上述三个主流平台测试通过, 熟悉其他语言的同学可以发现这基本上是开箱即用** + +- **持续更新中**... + +# 编译和测试使用 +**Note**: 下述所有步骤都在TDengine项目的根目录下进行 +**Note**: 请先确保src/connector/odbc如下所示,被包含在src/CMakeLists.txt源文件中 +``` +... +ADD_SUBDIRECTORY(dnode) +ADD_SUBDIRECTORY(connector/odbc) +ADD_SUBDIRECTORY(connector/jdbc) +``` + +# Linux下的编译, 以Ubuntu为例 +``` +sudo apt install unixodbc unixodbc-dev flex +rm -rf debug && cmake -B debug && cmake --build debug && cmake --install debug && echo yes +``` +# MacOSX下的编译, 以Catalina为例,依赖homebrew进行第三方工具安装[https://brew.sh/] +``` +brew install unixodbc +rm -rf debug && cmake -B debug && cmake --build debug && cmake --install debug && echo yes +``` +# Windows下的编译, 以Windows 10为例 +- 安装windows的`flex`工具. 目前我们使用[https://github.com/lexxmark/winflexbison](url). 安装完成后请确保win_flex.exe所在目录记录于`PATH`环境变量中. +- 安装Microsoft Visual Studio工具, 以VS2019为例 +- `"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"` +- `rmdir /s /q debug` +- `cmake -G "NMake Makefiles" -B debug` +- `cmake --build debug` +- `cmake --install debug` +- 以管理员身份打开`命令提示符` +- 安装ODBC驱动: 在上述打开的提示符下执行 `odbcconf /A {INSTALLDRIVER "TAOS | Driver=C:/TDengine/driver/todbc.dll | ConnectFunctions=YYN | DriverODBCVer=03.00"}` +- 新增一个数据源DSN: 执行 `odbcconf /A {CONFIGDSN "TAOS" "DSN=TAOS_DSN | Server=:"}` +上述步骤出现失败的话,可以参看这些链接: +1. win flex的安装: https://github.com/lexxmark/winflexbison/releases +2. PATH环境变量: https://jingyan.baidu.com/article/8ebacdf02d3c2949f65cd5d0.html +3. 管理员身份: https://blog.csdn.net/weixin_41083002/article/details/81019893 +4. 安装odbc驱动/数据源: https://docs.microsoft.com/en-us/sql/odbc/odbcconf-exe?view=sql-server-ver15 + +# 测试使用 +强烈建议您在linux上编译运行taosd服务端,因为当前TAOS还没有windows侧的服务端移植. +**Note1**: <>符号所括起的内容请按您当前的系统填写 +**Note2**: `.stmts` 文件存放的是测试用sql语句, 注意其格式为`UTF-8`(不带BOM导引头) +## 按官方文档在linux侧启动taosd,确保选用'UTF-8'作为其字符集 +## 在linux下创建数据 +``` +./debug/build/bin/tcodbc --dsn TAOS_DSN --uid --pwd --sts ./src/connector/odbc/samples/create_data.stmts +--<或指定特殊的ODBC连接字符串 --> +./debug/build/bin/tcodbc -C 'DSN=TAOS_DSN;UID=;PWD=;Server=:' --sts ./src/connector/odbc/samples/create_data.stmts +``` +## 在windows下检索数据 +``` +.\debug\build\bin\tcodbc -C "DSN=TAOS_DSN;UID=;PWD=;Server=:;enc_char=UTF-8" --sts .\src\connector\odbc\samples\query_data.stmts +``` +## 在MacOSX下检索数据 +``` +./debug/build/bin/tcodbc -C "DSN=TAOS_DSN;UID=;PWD=;Server=:" --sts ./src/connector/odbc/samples/query_data.stmts +``` + +## 代码示例 +- src/connector/odbc/examples/c +- src/connector/odbc/examples/js +- src/connector/odbc/examples/py +- src/connector/odbc/examples/rust +- src/connector/odbc/examples/go + +在linux或MacOSX上, 可以通过修改运行如下脚本来尝试各种测试: +**Note**: 不要忘记替换: +**Note**: 你需要在你的平台上安装nodejs/python/rust/go +**Note**: 你还需要安装对应语言的ODBC包: +-- node-odbc for nodejs: https://www.npmjs.com/package/odbc +-- pyodbc for python: https://pypi.org/project/pyodbc/ +-- rust-odbc for rust: https://docs.rs/odbc/0.17.0/odbc/ +-- go-odbc for go: https://github.com/alexbrainman/odbc + +``` +echo c && +./debug/build/bin/tcodbc -C "DSN=TAOS_DSN;Server=:" --sts src/connector/odbc/samples/create_data.stmts && +echo nodejs && +./src/connector/odbc/examples/js/odbc.js -C 'DSN=TAOS_DSN;Server=:' && +echo python && +python3 src/connector/odbc/examples/py/odbc.py -C 'DSN=TAOS_DSN;Server=:' && +echo rust && +pushd src/connector/odbc/examples/rust/main && DSN='DSN=TAOS_DSN;Server=:' cargo run && popd && +echo go && +DSN='DSN=TAOS_DSN;Server=:' go run src/connector/odbc/examples/go/odbc.go && +``` + +## 您可以对比测试一下prepared-batch-insert是否会带来速度的提升: +**注** src/connector/odbc/examples/c/main.c是tcodbc的源代码 +``` +./debug/build/bin/tcodbc -C "DSN=TAOS_DSN;Server=:" --insert --batch_size 200 --batchs 10000 +``` + + diff --git a/src/connector/odbc/README.md b/src/connector/odbc/README.md index e026884a0766772ac315acd3d0cac6535fb77557..670c7b9d9572ae066c51b2b43f593bed161fae40 100644 --- a/src/connector/odbc/README.md +++ b/src/connector/odbc/README.md @@ -1,20 +1,25 @@ # ODBC Driver # -- **very initial implementation of ODBC driver for TAOS +- **on-going implementation of ODBC driver for TAOS** -- **currently partially supported ODBC functions are: ` +- **currently exported ODBC functions are**: SQLAllocEnv SQLFreeEnv SQLAllocConnect SQLFreeConnect +SQLGetEnvAttr +SQLSetEnvAttr +SQLGetConnectAttr +SQLGetConnectOption +SQLGetInfo SQLConnect SQLDisconnect SQLAllocStmt SQLAllocHandle +SQLFreeHandle SQLFreeStmt SQLExecDirect -SQLExecDirectW SQLNumResultCols SQLRowCount SQLColAttribute @@ -22,29 +27,62 @@ SQLGetData SQLFetch SQLPrepare SQLExecute -SQLGetDiagField +SQLParamData +SQLPutData SQLGetDiagRec SQLBindParameter +SQLDescribeParam SQLDriverConnect SQLSetConnectAttr SQLDescribeCol +SQLBindCol SQLNumParams SQLSetStmtAttr -ConfigDSN +SQLBindParam +SQLCancel +SQLCancelHandle +SQLCloseCursor +SQLColumns +SQLCopyDesc +SQLDataSources +SQLEndTran +SQLFetchScroll +SQLGetCursorName +SQLGetDescField +SQLGetDescRec +SQLGetStmtAttr +SQLGetStmtOption +SQLGetTypeInfo +SQLSetConnectOption +SQLSetCursorName +SQLSetDescField +SQLSetDescRec +SQLSetParam +SQLSetStmtOption +SQLSpecialColumns +SQLStatistics +SQLTables +SQLTransact + ` -- **internationalized, you can specify different charset/code page for easy going. eg.: insert `utf-8.zh_cn` characters into database located in linux machine, while query them out in `gb2312/gb18030/...` code page in your chinese windows machine, or vice-versa. and much fun, insert `gb2312/gb18030/...` characters into database located in linux box from -your japanese windows box, and query them out in your local chinese windows machine. +- **internationalized, you can specify charset for SQLCHAR/SQLWCHAR/taos_charset/system-locale to coordinate with the environment**. -- **enable ODBC-aware software to communicate with TAOS. +- **enable ODBC-aware software to communicate with TAOS, no matter what platform it's running on, currently we support linux/macosx/windows** -- **enable any language with ODBC-bindings/ODBC-plugings to communicate with TAOS +- **enable any language with ODBC-bindings/ODBC-plugings to communicate with TAOS, currently c/nodejs/python/rust/go are all passed in our test environment, we believe other languages with ODBC-bindings/plugins are available-out-of-box** -- **still going on... +- **still going on**... # Building and Testing **Note**: all `work` is done in TDengine's project directory - +**Note**: please make sure src/connector/odbc is included in src/CMakeLists.txt +``` +... +ADD_SUBDIRECTORY(dnode) +ADD_SUBDIRECTORY(connector/odbc) +ADD_SUBDIRECTORY(connector/jdbc) +``` # Building under Linux, use Ubuntu as example ``` @@ -53,36 +91,68 @@ rm -rf debug && cmake -B debug && cmake --build debug && cmake --install debug & ``` # Building under Windows, use Windows 10 as example - install windows `flex` port. We use [https://github.com/lexxmark/winflexbison](url) at the moment. Please be noted to append `` to your `PATH`. -- install Microsoft Visual Studio, take VS2015 as example here -- `"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64` +- install Microsoft Visual Studio, take VS2019 as example here +- `"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"` - `rmdir /s /q debug` - `cmake -G "NMake Makefiles" -B debug` - `cmake --build debug` - `cmake --install debug` - open your `Command Prompt` with Administrator's priviledge -- remove previously installed TAOS ODBC driver: run `C:\TDengine\todbcinst -u -f -n TAOS` -- install TAOS ODBC driver that was just built: run `C:\TDengine\todbcinst -i -n TAOS -p C:\TDengine\driver` -- add a new user dsn: run `odbcconf CONFIGDSN TAOS "DSN=TAOS_DSN|Server=:` +- install TAOS ODBC driver that was just built: run `odbcconf /A {INSTALLDRIVER "TAOS | Driver=C:/TDengine/driver/todbc.dll | ConnectFunctions=YYN | DriverODBCVer=03.00"}` +- add a new user dsn: run `odbcconf /A {CONFIGDSN "TAOS" "DSN=TAOS_DSN | Server=host:port"}` # Test -we highly suggest that you build both in linux(ubuntu) and windows(windows 10) platform, because currently TAOS only has it's server-side port on linux platform. +we highly suggest that you build both in linux(ubuntu) and windows(windows 10) platform, because currently TAOS has not server-side port on windows platform. **Note1**: content within <> shall be modified to match your environment **Note2**: `.stmts` source files are all encoded in `UTF-8` -## start taosd in linux, suppose charset is `UTF-8` as default -``` -taosd -c ./debug/test/cfg -``` +## start taosd in linux, suppose charset is `UTF-8` as default, please follow TAOS doc for starting up ## create data in linux ``` -./debug/build/bin/tcodbc --dsn TAOS_DSN --uid --pwd --sts ./src/connector/odbc/tests/create_data.stmts +./debug/build/bin/tcodbc --dsn TAOS_DSN --uid --pwd --sts ./src/connector/odbc/samples/create_data.stmts -- -./debug/build/bin/tcodbc --dcs 'Driver=TAOS;UID=;PWD=;Server=:;client_enc=UTF-8' ./src/connector/odbc/tests/create_data.stmts +./debug/build/bin/tcodbc -C 'DSN=TAOS_DSN;UID=;PWD=;Server=:' --sts ./src/connector/odbc/samples/create_data.stmts ``` ## query data in windows ``` -.\debug\build\bin\tcodbc --dsn TAOS_DSN --uid --pwd --sts .\src\connector\odbc\tests\query_data.stmts --- -.\debug\build\bin\tcodbc --dcs "Driver=TAOS;UID=;PWD=;Server=:;client_enc=UTF-8" .\src\connector\odbc\tests\query_data.stmts +.\debug\build\bin\tcodbc -C "DSN=TAOS_DSN;UID=;PWD=;Server=:;enc_char=UTF-8" --sts .\src\connector\odbc\samples\query_data.stmts +``` +## query data in MacOSX +``` +./debug/build/bin/tcodbc -C "DSN=TAOS_DSN;UID=;PWD=;Server=:" --sts ./src/connector/odbc/samples/query_data.stmts +``` + +## code examples +- src/connector/odbc/examples/c +- src/connector/odbc/examples/js +- src/connector/odbc/examples/py +- src/connector/odbc/examples/rust +- src/connector/odbc/examples/go + +on linux/MacOSX, here after are script-snippet for you to play with: +**Note**: don't forget to replace : with whatever on your environment +**Note**: you need to install node/python3/rust/go on you machine +**Note**: you also need to install odbc-bindings/odbc-pluggins on those language platform, such as: +-- node-odbc for nodejs: https://www.npmjs.com/package/odbc +-- pyodbc for python: https://pypi.org/project/pyodbc/ +-- rust-odbc for rust: https://docs.rs/odbc/0.17.0/odbc/ +-- go-odbc for go: https://github.com/alexbrainman/odbc + +``` +echo c && +./debug/build/bin/tcodbc -C "DSN=TAOS_DSN;Server=:" --sts src/connector/odbc/samples/create_data.stmts && +echo nodejs && +./src/connector/odbc/examples/js/odbc.js -C 'DSN=TAOS_DSN;Server=:' && +echo python && +python3 src/connector/odbc/examples/py/odbc.py -C 'DSN=TAOS_DSN;Server=:' && +echo rust && +pushd src/connector/odbc/examples/rust/main && DSN='DSN=TAOS_DSN;Server=:' cargo run && popd && +echo go && +DSN='DSN=TAOS_DSN;Server=:' go run src/connector/odbc/examples/go/odbc.go && +``` + +## see how fast prepared-statment could bring up with: +``` +./debug/build/bin/tcodbc -C "DSN=TAOS_DSN;Server=:" --insert --batch_size 200 --batchs 10000 ``` diff --git a/src/connector/odbc/examples/CMakeLists.txt b/src/connector/odbc/examples/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..71f00a04e3013bc35137629555bdaa037850f0b1 --- /dev/null +++ b/src/connector/odbc/examples/CMakeLists.txt @@ -0,0 +1,4 @@ +PROJECT(TDengine) + +ADD_SUBDIRECTORY(c) + diff --git a/src/connector/odbc/examples/c/CMakeLists.txt b/src/connector/odbc/examples/c/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..7821f894d072e129be563805a10901e403e3cf2a --- /dev/null +++ b/src/connector/odbc/examples/c/CMakeLists.txt @@ -0,0 +1,22 @@ +PROJECT(TDengine) + +ADD_EXECUTABLE(tcodbc main.c ../../src/todbc_log.c) + +IF (TD_LINUX OR TD_DARWIN) + TARGET_LINK_LIBRARIES(tcodbc taos odbc) +ENDIF () + +IF (TD_DARWIN) + target_include_directories(tcodbc PRIVATE /usr/local/include) + target_link_directories(tcodbc PUBLIC /usr/local/lib) +ENDIF () + +IF (TD_WINDOWS_64) + SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /GL") + SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /GL") + TARGET_LINK_LIBRARIES(tcodbc taos_static odbc32 odbccp32 user32 legacy_stdio_definitions os) + + ADD_EXECUTABLE(tms main.cpp) + TARGET_LINK_LIBRARIES(tms odbc32) +ENDIF () + diff --git a/src/connector/odbc/examples/c/main.c b/src/connector/odbc/examples/c/main.c new file mode 100644 index 0000000000000000000000000000000000000000..e36c75688e3440a62b66fa5fc2f8b13b83f55237 --- /dev/null +++ b/src/connector/odbc/examples/c/main.c @@ -0,0 +1,1060 @@ +#include "../../src/todbc_log.h" + +#ifdef _MSC_VER +#include +#include +#include "os.h" +#endif +#include +#include +#include + +#include "taos.h" +#include "taoserror.h" + +#include +#include + +#define CHK_TEST(statement) \ +do { \ + D("testing: %s", #statement); \ + int r = (statement); \ + if (r) { \ + D("testing failed: %s", #statement); \ + return 1; \ + } \ +} while (0); + +typedef struct { + int batch_size; + int batchs; + int keep_stmt_among_batchs; + int use_odbc; + int use_taos_query; + int use_taos_stmt; +} insert_arg_t; + +typedef struct db_column_s db_column_t; +struct db_column_s { + SQLSMALLINT nameLength; + char name[4096]; // seems enough + SQLSMALLINT dataType; + SQLULEN columnSize; + SQLSMALLINT decimalDigits; + SQLSMALLINT nullable; +}; + +static db_column_t *columns = NULL; + +typedef struct data_s data_t; +struct data_s { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40+1]; + char blob[40+1]; // why 80? ref: tests/examples/c/apitest.c +}; + +#define CHK_RESULT(r, ht, h, fmt, ...) \ +do { \ + if (r==0) break; \ + SQLSMALLINT i_0381 = 1; \ + while (1) { \ + SQLCHAR ss[10]; \ + SQLINTEGER ne = 0; \ + SQLCHAR es[4096]; \ + SQLSMALLINT n = 0; \ + ss[0] = '\0'; \ + es[0] = '\0'; \ + SQLRETURN ret = SQLGetDiagRec(ht, h, i_0381, ss, &ne, es, sizeof(es), &n); \ + if (ret) break; \ + D("[%s]%s: " fmt "", ss, es, ##__VA_ARGS__); \ + ++i_0381; \ + } \ +} while (0) + +static int open_connect(const char *dsn, const char *uid, const char *pwd, SQLHENV *pEnv, SQLHDBC *pConn) { + SQLRETURN r; + SQLHENV env = {0}; + SQLHDBC conn = {0}; + r = SQLAllocEnv(&env); + if (r!=SQL_SUCCESS) return 1; + do { + r = SQLAllocConnect(env, &conn); + CHK_RESULT(r, SQL_HANDLE_ENV, env, ""); + if (r!=SQL_SUCCESS) break; + do { + r = SQLConnect(conn, (SQLCHAR*)dsn, (SQLSMALLINT)(dsn ? strlen(dsn) : 0), + (SQLCHAR*)uid, (SQLSMALLINT)(uid ? strlen(uid) : 0), + (SQLCHAR*)pwd, (SQLSMALLINT)(pwd ? strlen(pwd) : 0)); + CHK_RESULT(r, SQL_HANDLE_DBC, conn, ""); + if (r==SQL_SUCCESS) { + *pEnv = env; + *pConn = conn; + return 0; + } + } while (0); + SQLFreeConnect(conn); + } while (0); + SQLFreeEnv(env); + + return 1; +} + +static int open_driver_connect(const char *connstr, SQLHENV *pEnv, SQLHDBC *pConn) { + SQLRETURN r; + SQLHENV env = {0}; + SQLHDBC conn = {0}; + r = SQLAllocEnv(&env); + if (r!=SQL_SUCCESS) return 1; + do { + r = SQLAllocConnect(env, &conn); + CHK_RESULT(r, SQL_HANDLE_ENV, env, ""); + if (r!=SQL_SUCCESS) break; + do { + SQLCHAR buf[4096]; + SQLSMALLINT blen = 0; + SQLHDBC ConnectionHandle = conn; + SQLHWND WindowHandle = NULL; + SQLCHAR * InConnectionString = (SQLCHAR*)connstr; + SQLSMALLINT StringLength1 = (SQLSMALLINT)(connstr ? strlen(connstr) : 0); + SQLCHAR * OutConnectionString = buf; + SQLSMALLINT BufferLength = sizeof(buf); + SQLSMALLINT * StringLength2Ptr = &blen; + SQLUSMALLINT DriverCompletion = SQL_DRIVER_NOPROMPT; + r = SQLDriverConnect(ConnectionHandle, WindowHandle, InConnectionString, + StringLength1, OutConnectionString, BufferLength, + StringLength2Ptr, DriverCompletion); + CHK_RESULT(r, SQL_HANDLE_DBC, conn, ""); + if (r==SQL_SUCCESS) { + *pEnv = env; + *pConn = conn; + return 0; + } + } while (0); + SQLFreeConnect(conn); + } while (0); + SQLFreeEnv(env); + + return 1; +} + +static SQLRETURN traverse_cols(SQLHSTMT stmt, SQLSMALLINT cols) { + SQLRETURN r = SQL_ERROR; + for (SQLSMALLINT i=0; i0) fprintf(stdout, "\n"); + return r; + } + } + if (soi==SQL_NULL_DATA) { + fprintf(stdout, "%snull", i==0?"":","); + } else { + fprintf(stdout, "%s\"%s\"", i==0?"":",", buf); + } + } + fprintf(stdout, "\n"); + } + } while (0); + return r; +} + +int test_statements(const char *dsn, const char *uid, const char *pwd, const char **statements) { + SQLRETURN r = SQL_SUCCESS; + SQLHENV env = {0}; + SQLHDBC conn = {0}; + int n = open_connect(dsn, uid, pwd, &env, &conn); + if (n) return 1; + do { + SQLHSTMT stmt = {0}; + r = SQLAllocHandle(SQL_HANDLE_STMT, conn, &stmt); + if (r!=SQL_SUCCESS) break; + const char **p = statements; + while (*p) { + if (do_statement(stmt, *p)) { + r = SQL_ERROR; + break; + } + ++p; + } + SQLFreeHandle(SQL_HANDLE_STMT, stmt); + } while (0); + SQLDisconnect(conn); + SQLFreeConnect(conn); + SQLFreeEnv(env); + return r ? 1 : 0; +} + +int test_driver_connect(const char *connstr) { + SQLRETURN r = SQL_SUCCESS; + SQLHENV env = {0}; + SQLHDBC conn = {0}; + int n = open_driver_connect(connstr, &env, &conn); + if (n) return 1; + SQLDisconnect(conn); + SQLFreeConnect(conn); + SQLFreeEnv(env); + return r ? 1 : 0; +} + +int create_statement(SQLHENV env, SQLHDBC conn, SQLHSTMT *pStmt) { + SQLHSTMT stmt = {0}; + SQLRETURN r = SQLAllocHandle(SQL_HANDLE_STMT, conn, &stmt); + CHK_RESULT(r, SQL_HANDLE_DBC, conn, ""); + if (r==SQL_SUCCESS) { + *pStmt = stmt; + return 0; + } + if (r==SQL_SUCCESS_WITH_INFO) { + SQLFreeHandle(SQL_HANDLE_STMT, stmt); + } + return 1; +} + +int do_statements(SQLHSTMT stmt, const char **statements) { + const char **p = statements; + while (p && *p) { + CHK_TEST(do_statement(stmt, *p)); + ++p; + } + return 0; +} + +int tests_stmt(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt) { + const char *statements[] = { + "drop database if exists m", + "create database m", + "use m", + // "create table t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, blob binary(1), name nchar(1))", + "create table t (ts timestamp, b bool)", + "insert into t values('2020-10-10 00:00:00', 0)", + "insert into t values('2020-10-10 00:00:00.001', 1)", + NULL + }; + CHK_TEST(do_statements(stmt, statements)); + return 0; +} + +int tests(SQLHENV env, SQLHDBC conn) { + SQLHSTMT stmt = {0}; + CHK_TEST(create_statement(env, conn, &stmt)); + int r = tests_stmt(env, conn, stmt); + SQLFreeHandle(SQL_HANDLE_STMT, stmt); + return r ? 1 : 0; +} + +int test_env(void) { + SQLRETURN r; + SQLHENV env = {0}; + r = SQLAllocEnv(&env); + if (r!=SQL_SUCCESS) return 1; + SQLFreeEnv(env); + return 0; +} + +static int test_sqls_in_stmt(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt, const char *sqls) { + FILE *f = fopen(sqls, "rb"); + if (!f) { + D("failed to open file [%s]", sqls); + return -1; + } + + int r = 0; + while (!feof(f)) { + char *line = NULL; + size_t len = 0; + + ssize_t n = 0; +#ifdef _MSC_VER + n = taosGetlineImp(&line, &len, f); +#else + n = getline(&line, &len, f); +#endif + if (n==-1) break; + + const char *p = NULL; + do { + if (line[0] == '#') break; + if (n>0 && line[n-1] == '\n') line[n-1]='\0'; + if (n>0 && line[n-1] == '\r') line[n-1]='\0'; + if (n>1 && line[n-2] == '\r') line[n-2]='\0'; + p = line; + while (isspace(*p)) ++p; + + if (*p==0) break; + + int positive = 1; + if (strncmp(p, "N:", 2)==0) { + // negative sample + positive = 0; + p += 2; + } else if (strncmp(p, "P:", 2)==0) { + // positive sample + p += 2; + } + + D("statement: [%s]", p); + r = do_statement(stmt, p); + + if (positive && r==0) break; + if (!positive && r) { r = 0; break; } + if (positive) return r; + D("expecting negative result, but got positive"); + return -1; + } while (0); + + free(line); + + if (r) break; + } + + fclose(f); + return r ? 1 : 0; +} + +static int test_sqls_in_conn(SQLHENV env, SQLHDBC conn, const char *sqls) { + SQLHSTMT stmt = {0}; + CHK_TEST(create_statement(env, conn, &stmt)); + int r = test_sqls_in_stmt(env, conn, stmt, sqls); + SQLFreeHandle(SQL_HANDLE_STMT, stmt); + return r ? 1 : 0; +} + +static int test_sqls(const char *dsn, const char *uid, const char *pwd, const char *connstr, const char *sqls) { + int r = 0; + SQLHENV env = {0}; + SQLHDBC conn = {0}; + if (dsn) { + CHK_TEST(open_connect(dsn, uid, pwd, &env, &conn)); + } else { + CHK_TEST(open_driver_connect(connstr, &env, &conn)); + } + + if (sqls) { + r = test_sqls_in_conn(env, conn, sqls); + } + SQLDisconnect(conn); + SQLFreeConnect(conn); + SQLFreeEnv(env); + return r ? 1 : 0; +} + +typedef struct record_s record_t; +struct record_s { + int dummy; + char ts[64]; + SQLLEN ts_len; + int32_t v1; + SQLLEN v1_len; + char ts2[64]; + SQLLEN ts2_len; +}; + +static int do_prepare_in_stmt(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt) { + SQLRETURN r = SQL_SUCCESS; + do { + const char *sql = "insert into m.v (ts, v1, ts2) values (?, ?, ?)"; + r = SQLPrepare(stmt, (SQLCHAR*)sql, SQL_NTS); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + + record_t records[] = { + {0, "2020-01-03 11:22:33.345", SQL_NTS, 1, sizeof(int32_t), "2020-01-02 11:22:33.455", SQL_NTS}, + {0, "2020-01-03 11:22:34.346", SQL_NTS, 2, sizeof(int32_t), "2020-01-02 11:22:34.445", SQL_NTS}, + {0, "2020-01-04 11:22:34.345", SQL_NTS, 2, sizeof(int32_t), "2020-01-02 11:22:34.445", SQL_NTS}, + {0, "2020-01-05 11:22:34.345", SQL_NTS, 2, sizeof(int32_t), "2020-01-02 11:22:34.445", SQL_NTS}, + }; + + record_t *base = (record_t*)0; + + r = SQLBindParameter(stmt, 1, SQL_PARAM_INPUT, SQL_C_CHAR, SQL_CHAR, sizeof(base->ts)-1, 0, base->ts, sizeof(base->ts), &(base->ts_len)); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + r = SQLBindParameter(stmt, 2, SQL_PARAM_INPUT, SQL_C_LONG, SQL_INTEGER, 0, 0, &base->v1, 0, &(base->v1_len)); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + r = SQLBindParameter(stmt, 3, SQL_PARAM_INPUT, SQL_C_CHAR, SQL_CHAR, sizeof(base->ts2)-1, 0, base->ts2, sizeof(base->ts2), &(base->ts2_len)); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + + SQLSetStmtAttr(stmt, SQL_ATTR_PARAM_BIND_TYPE, (SQLPOINTER)sizeof(*base), 0); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + + SQLSetStmtAttr(stmt, SQL_ATTR_PARAMSET_SIZE, (SQLPOINTER)(sizeof(records)/sizeof(records[0])), 0); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + + record_t *record = NULL; + + SQLSetStmtAttr(stmt, SQL_ATTR_PARAM_BIND_OFFSET_PTR, &record, 0); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + + record = records; + + r = SQLExecute(stmt); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + } while (0); + + return r ? -1 : 0; +} + +static int do_prepare_in_conn(SQLHENV env, SQLHDBC conn) { + SQLHSTMT stmt = {0}; + CHK_TEST(create_statement(env, conn, &stmt)); + int r = do_prepare_in_stmt(env, conn, stmt); + SQLFreeHandle(SQL_HANDLE_STMT, stmt); + return r ? 1 : 0; +} + +static int do_prepare(const char *dsn, const char *uid, const char *pwd, const char *connstr) { + int r = 0; + SQLHENV env = {0}; + SQLHDBC conn = {0}; + if (dsn) { + CHK_TEST(open_connect(dsn, uid, pwd, &env, &conn)); + } else { + CHK_TEST(open_driver_connect(connstr, &env, &conn)); + } + + r = do_prepare_in_conn(env, conn); + + SQLDisconnect(conn); + SQLFreeConnect(conn); + SQLFreeEnv(env); + return r ? 1 : 0; +} + +typedef struct { + int dummy; + int64_t ts; + SQLLEN ts_len; + int8_t v1; + SQLLEN v1_len; + int16_t v2; + SQLLEN v2_len; + int32_t v4; + SQLLEN v4_len; + int64_t v8; + SQLLEN v8_len; +} test_v_t; + +static int do_insert_in_stmt(SQLHENV env, SQLHDBC conn, SQLHSTMT stmt, int64_t *ts, insert_arg_t *arg) { + SQLRETURN r = SQL_SUCCESS; + int batch_size = arg->batch_size; + test_v_t *recs = NULL; + do { + const char *sql = "insert into test.v (ts, v1, v2, v4, v8) values (?, ?, ?, ?, ?)"; + r = SQLPrepare(stmt, (SQLCHAR*)sql, SQL_NTS); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + + test_v_t *base = NULL; + + r = SQLBindParameter(stmt, 1, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_BIGINT, 0, 0, &base->ts, 0, &base->ts_len); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + r = SQLBindParameter(stmt, 2, SQL_PARAM_INPUT, SQL_C_TINYINT, SQL_TINYINT, 0, 0, &base->v1, 0, &base->v1_len); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + r = SQLBindParameter(stmt, 3, SQL_PARAM_INPUT, SQL_C_SHORT, SQL_SMALLINT, 0, 0, &base->v2, 0, &base->v2_len); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + r = SQLBindParameter(stmt, 4, SQL_PARAM_INPUT, SQL_C_LONG, SQL_INTEGER, 0, 0, &base->v4, 0, &base->v4_len); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + r = SQLBindParameter(stmt, 5, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_BIGINT, 0, 0, &base->v8, 0, &base->v8_len); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + + SQLSetStmtAttr(stmt, SQL_ATTR_PARAM_BIND_TYPE, (SQLPOINTER)sizeof(*base), 0); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + + base = NULL; + + SQLSetStmtAttr(stmt, SQL_ATTR_PARAM_BIND_OFFSET_PTR, &base, 0); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + + size_t n_recs = (size_t)batch_size; + recs = (test_v_t*)calloc(n_recs, sizeof(*recs)); + OILE(recs, ""); + + SQLSetStmtAttr(stmt, SQL_ATTR_PARAMSET_SIZE, (SQLPOINTER)n_recs, 0); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + + base = recs; + + for (int batch=0; batchbatchs; ++batch) { + for (int i=0; idummy = 0; + rec->ts = *ts + i; + rec->ts_len = sizeof(rec->ts); + rec->v1 = (int8_t)rand(); + rec->v1_len = sizeof(rec->v1); + rec->v2 = (int16_t)rand(); + rec->v2_len = sizeof(rec->v2); + rec->v4 = rand(); + rec->v4_len = sizeof(rec->v4); + rec->v8 = rand(); + rec->v8_len = sizeof(rec->v8); + } + + *ts += (int64_t)n_recs; + + r = SQLExecute(stmt); + CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); + if (r) break; + } + } while (0); + + free(recs); + return r ? -1 : 0; +} + +static int do_insert_in_conn(SQLHENV env, SQLHDBC conn, insert_arg_t *arg) { + SQLHSTMT stmt = {0}; + int64_t ts = 1502535178128; + int r = 0; + CHK_TEST(create_statement(env, conn, &stmt)); + for (int i=0; i<1 && ibatchs; ++i) { + r = do_insert_in_stmt(env, conn, stmt, &ts, arg); + if (r) break; + if (!arg->keep_stmt_among_batchs) { + SQLFreeHandle(SQL_HANDLE_STMT, stmt); + r = create_statement(env, conn, &stmt); + if (r) break; + } + } + SQLFreeHandle(SQL_HANDLE_STMT, stmt); + return r ? 1 : 0; +} + +static int do_insert_batch(const char *dsn, const char *uid, const char *pwd, const char *connstr, insert_arg_t *arg, const char *sqls[]) { + int r = 0; + SQLHENV env = {0}; + SQLHDBC conn = {0}; + if (dsn) { + CHK_TEST(open_connect(dsn, uid, pwd, &env, &conn)); + } else { + CHK_TEST(open_driver_connect(connstr, &env, &conn)); + } + + SQLHSTMT stmt = {0}; + CHK_TEST(create_statement(env, conn, &stmt)); + CHK_TEST(do_statements(stmt, sqls)); + SQLFreeHandle(SQL_HANDLE_STMT, stmt); + + OD("................"); + r = do_insert_in_conn(env, conn, arg); + OD("................"); + + SQLDisconnect(conn); + SQLFreeConnect(conn); + SQLFreeEnv(env); + return r ? 1 : 0; +} + +static int inited = 0; +static void init_once(void) { + if (inited) return; + + int r = taos_init(); + if (r) OILE(0, ""); + inited = 1; +} + +static int do_sqls(TAOS *taos, const char *sqls[]) { + for (int i=0; sqls[i]; ++i) { + OD("[%s]", sqls[i]); + TAOS_RES *res = taos_query(taos, sqls[i]); + if (!res) { + int e = terrno; + OD("taos_query [%s] failed: [%d]%s", sqls[i], e, tstrerror(e)); + return -1; + } + int e = taos_errno(res); + if (e) { + OD("taos_query [%s] failed: [%d]%s", sqls[i], e, tstrerror(e)); + } + taos_stop_query(res); + if (e) return -1; + } + return 0; +} + +static int do_taos_query(TAOS *taos, insert_arg_t *arg) { + char **sqls = (char**)calloc((size_t)arg->batchs, sizeof(*sqls)); + if (!sqls) { + OILE(0, "out of memory"); + } + + int64_t ts = 1502535178128; + for (int i=0; ibatchs; ++i) { + size_t bytes = 100 * (size_t)arg->batch_size; + sqls[i] = (char*)malloc(bytes); + OILE(sqls[i], ""); + char *p = sqls[i]; + size_t count = 0; + + while (1) { + int n = 0; + n = snprintf(p, bytes, "insert into test.v values"); + OILE(n>0, ""); + if (p) p += n; + OILE(bytes>n, ""); + if (bytes>=n) bytes -= (size_t)n; + else bytes = 0; + count += (size_t)n; + + for (int j=0; jbatch_size; ++j) { + int8_t v1 = (int8_t)rand(); if (v1==INT8_MIN) v1++; + int16_t v2 = (int16_t)rand(); if (v2==INT16_MIN) v2++; + int32_t v4 = (int32_t)rand(); if (v4==INT32_MIN) v4++; + int64_t v8 = (int64_t)rand(); if (v8==INT64_MIN) v8++; + n = snprintf(p, bytes, " (%" PRId64 ", %d, %d, %d, %" PRId64 ")", ts + i*arg->batch_size + j, (int)v1, (int)v2, v4, v8); + OILE(n>0, ""); + if (p) p += n; + OILE(bytes>n, ""); + if (bytes>=n) bytes -= (size_t)n; + else bytes = 0; + count += (size_t)n; + } + + if (p) break; + OILE(0, ""); + } + } + + OD(".............."); + for (int i=0; ibatchs; ++i) { + TAOS_RES *res = taos_query(taos, sqls[i]); + if (!res) { + int e = terrno; + OD("taos_query [%s] failed: [%d]%s", sqls[i], e, tstrerror(e)); + return -1; + } + int e = taos_errno(res); + if (e) { + OD("taos_query [%s] failed: [%d]%s", sqls[i], e, tstrerror(e)); + } + taos_stop_query(res); + if (e) return -1; + } + OD(".............."); + + for (int i=0; ibatchs; ++i) { + free(sqls[i]); + } + free(sqls); + + return 0; +} + +static int do_taos_stmt(TAOS *taos, insert_arg_t *arg) { + TAOS_STMT *stmt = taos_stmt_init(taos); + OILE(stmt, ""); + const char *sql = "insert into test.v values (?,?,?,?,?)"; + int r = 0; + do { + r = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); + if (r) { + OD("taos_stmt_prepare [%s] failed: [%d]%s", sql, r, tstrerror(r)); + break; + } + int64_t ts = 1502535178128; + TAOS_BIND *bindings = (TAOS_BIND*)calloc(5, sizeof(*bindings)); + TAOS_BIND *b_ts = bindings + 0; + TAOS_BIND *b_v1 = bindings + 1; + TAOS_BIND *b_v2 = bindings + 2; + TAOS_BIND *b_v4 = bindings + 3; + TAOS_BIND *b_v8 = bindings + 4; + b_ts->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + b_ts->buffer_length = sizeof(b_ts->u.ts); + b_ts->length = &b_ts->buffer_length; + b_ts->buffer = &b_ts->u.ts; + b_ts->is_null = NULL; + + b_v1->buffer_type = TSDB_DATA_TYPE_TINYINT; + b_v1->buffer_length = sizeof(b_v1->u.v1); + b_v1->length = &b_v1->buffer_length; + b_v1->buffer = &b_v1->u.v1; + b_v1->is_null = NULL; + + b_v2->buffer_type = TSDB_DATA_TYPE_SMALLINT; + b_v2->buffer_length = sizeof(b_v2->u.v2); + b_v2->length = &b_v2->buffer_length; + b_v2->buffer = &b_v2->u.v2; + b_v2->is_null = NULL; + + b_v4->buffer_type = TSDB_DATA_TYPE_INT; + b_v4->buffer_length = sizeof(b_v4->u.v4); + b_v4->length = &b_v4->buffer_length; + b_v4->buffer = &b_v4->u.v4; + b_v4->is_null = NULL; + + b_v8->buffer_type = TSDB_DATA_TYPE_BIGINT; + b_v8->buffer_length = sizeof(b_v8->u.v8); + b_v8->length = &b_v8->buffer_length; + b_v8->buffer = &b_v8->u.v8; + b_v8->is_null = NULL; + + OILE(bindings, ""); + OD("................"); + for (int i=0; ibatchs; ++i) { + for (int j=0; jbatch_size; ++j) { + b_ts->u.ts = ts + i*arg->batch_size + j; + b_v1->u.v1 = (int8_t)rand(); + b_v2->u.v2 = (int16_t)rand(); + b_v4->u.v4 = (int32_t)rand(); + b_v8->u.v8 = (int64_t)rand(); + r = taos_stmt_bind_param(stmt, bindings); + if (r) { + OD("taos_stmt_bind_param failed: [%d]%s", r, tstrerror(r)); + break; + } + r = taos_stmt_add_batch(stmt); + if (r) { + OD("taos_stmt_add_batch failed: [%d]%s", r, tstrerror(r)); + break; + } + } + + if (r) break; + + r = taos_stmt_execute(stmt); + if (r) { + OD("taos_stmt_execute failed: [%d]%s", r, tstrerror(r)); + break; + } + } + OD("................"); + + free(bindings); + + if (r) break; + } while (0); + taos_stmt_close(stmt); + return r ? -1 : 0; +} + +static int do_insert_batch_taos(const char *dsn, const char *uid, const char *pwd, const char *connstr, insert_arg_t *arg, const char *sqls[]) { + int r = 0; + + init_once(); + + int port = 0; + char *ip = NULL; + const char *p = strchr(connstr, ':'); + if (p) { + ip = strndup(connstr, (size_t)(p-connstr)); + ++p; + sscanf(p, "%d", &port); + } else { + ip = strdup(connstr); + port = 6030; + } + if (!ip) { + OD("bad ip/port:[%s]", connstr); + return -1; + } + + TAOS *taos = NULL; + do { + taos = taos_connect(ip, uid, pwd, NULL, (uint16_t)port); + if (!taos) { + int e = terrno; + OD("taos_connect [%s/%d] failed:[%d]%s", ip, port, e, tstrerror(e)); + break; + } + r = do_sqls(taos, sqls); + if (r) break; + if (arg->use_taos_query) { + r = do_taos_query(taos, arg); + } else if (arg->use_taos_stmt) { + r = do_taos_stmt(taos, arg); + } else { + OILE(0, ""); + } + } while (0); + + if (taos) taos_close(taos); + free(ip); + + return r ? 1 : 0; +} + +static int do_debug_col_name_max_len(const char *dsn, const char *uid, const char *pwd, const char *connstr) { + SQLRETURN r; + SQLHENV env = {0}; + SQLHDBC conn = {0}; + r = SQLAllocEnv(&env); + if (r!=SQL_SUCCESS) { + D("SQLAllocEnv failed"); + return 1; + }; + do { + r = SQLAllocConnect(env, &conn); + CHK_RESULT(r, SQL_HANDLE_ENV, env, ""); + if (r!=SQL_SUCCESS) break; + do { + if (dsn) { + r = SQLConnect(conn, (SQLCHAR*)dsn, (SQLSMALLINT)(dsn ? strlen(dsn) : 0), + (SQLCHAR*)uid, (SQLSMALLINT)(uid ? strlen(uid) : 0), + (SQLCHAR*)pwd, (SQLSMALLINT)(pwd ? strlen(pwd) : 0)); + } else { + SQLCHAR buf[4096]; + SQLSMALLINT blen = 0; + SQLHDBC ConnectionHandle = conn; + SQLHWND WindowHandle = NULL; + SQLCHAR * InConnectionString = (SQLCHAR*)connstr; + SQLSMALLINT StringLength1 = (SQLSMALLINT)(connstr ? strlen(connstr) : 0); + SQLCHAR * OutConnectionString = buf; + SQLSMALLINT BufferLength = sizeof(buf); + SQLSMALLINT * StringLength2Ptr = &blen; + SQLUSMALLINT DriverCompletion = SQL_DRIVER_NOPROMPT; + r = SQLDriverConnect(ConnectionHandle, WindowHandle, InConnectionString, + StringLength1, OutConnectionString, BufferLength, + StringLength2Ptr, DriverCompletion); + } + CHK_RESULT(r, SQL_HANDLE_DBC, conn, ""); + if (r!=SQL_SUCCESS) break; + D("connected"); + if (1) { + SQLSMALLINT maxColumnNameLength = 0; + SQLSMALLINT len = 0; + r = SQLGetInfo(conn, SQL_MAX_COLUMN_NAME_LEN, &maxColumnNameLength, sizeof(SQLSMALLINT), &len); + CHK_RESULT(r, SQL_HANDLE_DBC, conn, ""); + if (r!=SQL_SUCCESS) break; + D("maxColumnNameLength: %d", maxColumnNameLength); + } + } while (0); + SQLFreeConnect(conn); + conn = NULL; + } while (0); + SQLFreeEnv(env); + env = NULL; + + return (r==SQL_SUCCESS) ? 0 : 1; +} + +void usage(const char *arg0) { + fprintf(stdout, "%s usage:\n", arg0); + fprintf(stdout, "%s [--dsn ] [--uid ] [--pwd ] [-C ] [--sts ]\n", arg0); + fprintf(stdout, " --dsn : DSN\n"); + fprintf(stdout, " --uid : UID\n"); + fprintf(stdout, " --pwd : PWD\n"); + fprintf(stdout, " -C : driver connection string\n"); + fprintf(stdout, " --sts : file where statements store\n"); +} + +int main(int argc, char *argv[]) { + srand((unsigned)time(0)); + const char *conn_str = NULL; + const char *dsn = NULL; + const char *uid = NULL; + const char *pwd = NULL; + const char *sts = NULL; // statements file + int debug_col_name_max_len = 0; + int prepare = 0; + int insert = 0; + insert_arg_t insert_arg = { + .batch_size = 100, + .batchs = 100, + .keep_stmt_among_batchs = 0 + }; + for (size_t i=1; i=argc) { + D(" expected but got nothing"); + return 1; + } + sscanf(argv[i], "%d", &insert_arg.batch_size); + if (insert_arg.batch_size<=0) { + D(" invalid"); + return 1; + } + continue; + } + if (strcmp(arg, "--batchs")==0) { + ++i; + if (i>=argc) { + D(" expected but got nothing"); + return 1; + } + sscanf(argv[i], "%d", &insert_arg.batchs); + if (insert_arg.batchs<=0) { + D(" invalid"); + return 1; + } + continue; + } + if (strcmp(arg, "--keep_stmt_among_batchs")==0) { + insert_arg.keep_stmt_among_batchs = 1; + continue; + } + if (strcmp(arg, "--dsn")==0) { + ++i; + if (i>=argc) { + D(" expected but got nothing"); + return 1; + } + if (conn_str) { + D("-C has already been specified"); + return 1; + } + dsn = argv[i]; + continue; + } + if (strcmp(arg, "--uid")==0) { + ++i; + if (i>=argc) { + D(" expected but got nothing"); + return 1; + } + uid = argv[i]; + continue; + } + if (strcmp(arg, "--pwd")==0) { + ++i; + if (i>=argc) { + D(" expected but got nothing"); + return 1; + } + pwd = argv[i]; + continue; + } + if (strcmp(arg, "-C")==0) { + ++i; + if (i>=argc) { + D(" expected but got nothing"); + return 1; + } + if (dsn || uid || pwd) { + D("either of --dsn/--uid/--pwd has already been specified"); + return 1; + } + conn_str = argv[i]; + continue; + } + if (strcmp(arg, "--sts")==0) { + ++i; + if (i>=argc) { + D(" expected but got nothing"); + return 1; + } + sts = argv[i]; + continue; + } + if (strcmp(arg, "-p")==0) { + prepare = 1; + continue; + } + } + if (debug_col_name_max_len) { + int r = do_debug_col_name_max_len(dsn, uid, pwd, conn_str); + if (r) return 1; + } + if (insert) { + const char *sqls[] = { + "drop database if exists test", + "create database test", + "create table test.v (ts timestamp, v1 tinyint, v2 smallint, v4 int, v8 bigint)", + NULL + }; + int r = 0; + if (insert_arg.use_odbc) { + r = do_insert_batch(dsn, uid, pwd, conn_str, &insert_arg, sqls); + } else { + r = do_insert_batch_taos(dsn, uid, pwd, conn_str, &insert_arg, sqls); + } + if (r) return 1; + } + if (sts) { + int r = test_sqls(dsn, uid, pwd, conn_str, sts); + if (r) return 1; + } + if (prepare) { + int r = do_prepare(dsn, uid, pwd, conn_str); + if (r) return 1; + } + D("Done!"); + return 0; +} + diff --git a/src/connector/odbc/examples/c/main.cpp b/src/connector/odbc/examples/c/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b719534f986fd0076de7dc3351ce273a2847f919 --- /dev/null +++ b/src/connector/odbc/examples/c/main.cpp @@ -0,0 +1,654 @@ +/******************************************************************************* +/* ODBCSQL: a sample program that implements an ODBC command line interpreter. +/* +/* USAGE: ODBCSQL DSN= or +/* ODBCSQL FILEDSN= or +/* ODBCSQL DRIVER={driver name} +/* +/* +/* Copyright(c) Microsoft Corporation. This is a WDAC sample program and +/* is not suitable for use in production environments. +/* +/******************************************************************************/ +/* Modules: +/* Main Main driver loop, executes queries. +/* DisplayResults Display the results of the query if any +/* AllocateBindings Bind column data +/* DisplayTitles Print column titles +/* SetConsole Set console display mode +/* HandleError Show ODBC error messages +/******************************************************************************/ + +#define _UNICODE +#define UNICODE + +#include +#include +#include +#include +#include +#include +#include +#include + + +/*******************************************/ +/* Macro to call ODBC functions and */ +/* report an error on failure. */ +/* Takes handle, handle type, and stmt */ +/*******************************************/ + +#define TRYODBC(h, ht, x) { RETCODE rc = x;\ + if (rc != SQL_SUCCESS) \ + { \ + HandleDiagnosticRecord (h, ht, rc); \ + } \ + if (rc == SQL_ERROR) \ + { \ + fwprintf(stderr, L"Error in " L#x L"\n"); \ + goto Exit; \ + } \ + } +/******************************************/ +/* Structure to store information about */ +/* a column. +/******************************************/ + +typedef struct STR_BINDING { + SQLSMALLINT cDisplaySize; /* size to display */ + WCHAR *wszBuffer; /* display buffer */ + SQLLEN indPtr; /* size or null */ + BOOL fChar; /* character col? */ + struct STR_BINDING *sNext; /* linked list */ +} BINDING; + + + +/******************************************/ +/* Forward references */ +/******************************************/ + +void HandleDiagnosticRecord (SQLHANDLE hHandle, + SQLSMALLINT hType, + RETCODE RetCode); + +void DisplayResults(HSTMT hStmt, + SQLSMALLINT cCols); + +void AllocateBindings(HSTMT hStmt, + SQLSMALLINT cCols, + BINDING** ppBinding, + SQLSMALLINT* pDisplay); + + +void DisplayTitles(HSTMT hStmt, + DWORD cDisplaySize, + BINDING* pBinding); + +void SetConsole(DWORD cDisplaySize, + BOOL fInvert); + +/*****************************************/ +/* Some constants */ +/*****************************************/ + + +#define DISPLAY_MAX 50 // Arbitrary limit on column width to display +#define DISPLAY_FORMAT_EXTRA 3 // Per column extra display bytes (| ) +#define DISPLAY_FORMAT L"%c %*.*s " +#define DISPLAY_FORMAT_C L"%c %-*.*s " +#define NULL_SIZE 6 // +#define SQL_QUERY_SIZE 1000 // Max. Num characters for SQL Query passed in. + +#define PIPE L'|' + +SHORT gHeight = 80; // Users screen height + +int __cdecl wmain(int argc, _In_reads_(argc) WCHAR **argv) +{ + SQLHENV hEnv = NULL; + SQLHDBC hDbc = NULL; + SQLHSTMT hStmt = NULL; + WCHAR* pwszConnStr; + WCHAR wszInput[SQL_QUERY_SIZE]; + + // Allocate an environment + + if (SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &hEnv) == SQL_ERROR) + { + fwprintf(stderr, L"Unable to allocate an environment handle\n"); + exit(-1); + } + + // Register this as an application that expects 3.x behavior, + // you must register something if you use AllocHandle + + TRYODBC(hEnv, + SQL_HANDLE_ENV, + SQLSetEnvAttr(hEnv, + SQL_ATTR_ODBC_VERSION, + (SQLPOINTER)SQL_OV_ODBC3, + 0)); + + // Allocate a connection + TRYODBC(hEnv, + SQL_HANDLE_ENV, + SQLAllocHandle(SQL_HANDLE_DBC, hEnv, &hDbc)); + + if (argc > 1) + { + pwszConnStr = *++argv; + } + else + { + pwszConnStr = L""; + } + + // Connect to the driver. Use the connection string if supplied + // on the input, otherwise let the driver manager prompt for input. + + TRYODBC(hDbc, + SQL_HANDLE_DBC, + SQLDriverConnect(hDbc, + GetDesktopWindow(), + pwszConnStr, + SQL_NTS, + NULL, + 0, + NULL, + SQL_DRIVER_COMPLETE)); + + fwprintf(stderr, L"Connected!\n"); + + TRYODBC(hDbc, + SQL_HANDLE_DBC, + SQLAllocHandle(SQL_HANDLE_STMT, hDbc, &hStmt)); + + wprintf(L"Enter SQL commands, type (control)Z to exit\nSQL COMMAND>"); + + // Loop to get input and execute queries + + while(_fgetts(wszInput, SQL_QUERY_SIZE-1, stdin)) + { + RETCODE RetCode; + SQLSMALLINT sNumResults; + + // Execute the query + + if (!(*wszInput)) + { + wprintf(L"SQL COMMAND>"); + continue; + } + RetCode = SQLExecDirect(hStmt,wszInput, SQL_NTS); + + switch(RetCode) + { + case SQL_SUCCESS_WITH_INFO: + { + HandleDiagnosticRecord(hStmt, SQL_HANDLE_STMT, RetCode); + // fall through + } + case SQL_SUCCESS: + { + // If this is a row-returning query, display + // results + TRYODBC(hStmt, + SQL_HANDLE_STMT, + SQLNumResultCols(hStmt,&sNumResults)); + + if (sNumResults > 0) + { + DisplayResults(hStmt,sNumResults); + } + else + { + SQLLEN cRowCount; + + TRYODBC(hStmt, + SQL_HANDLE_STMT, + SQLRowCount(hStmt,&cRowCount)); + + if (cRowCount >= 0) + { + wprintf(L"%Id %s affected\n", + cRowCount, + cRowCount == 1 ? L"row" : L"rows"); + } + } + break; + } + + case SQL_ERROR: + { + HandleDiagnosticRecord(hStmt, SQL_HANDLE_STMT, RetCode); + break; + } + + default: + fwprintf(stderr, L"Unexpected return code %hd!\n", RetCode); + + } + TRYODBC(hStmt, + SQL_HANDLE_STMT, + SQLFreeStmt(hStmt, SQL_CLOSE)); + + wprintf(L"SQL COMMAND>"); + } + +Exit: + + // Free ODBC handles and exit + + if (hStmt) + { + SQLFreeHandle(SQL_HANDLE_STMT, hStmt); + } + + if (hDbc) + { + SQLDisconnect(hDbc); + SQLFreeHandle(SQL_HANDLE_DBC, hDbc); + } + + if (hEnv) + { + SQLFreeHandle(SQL_HANDLE_ENV, hEnv); + } + + wprintf(L"\nDisconnected."); + + return 0; + +} + +/************************************************************************ +/* DisplayResults: display results of a select query +/* +/* Parameters: +/* hStmt ODBC statement handle +/* cCols Count of columns +/************************************************************************/ + +void DisplayResults(HSTMT hStmt, + SQLSMALLINT cCols) +{ + BINDING *pFirstBinding, *pThisBinding; + SQLSMALLINT cDisplaySize; + RETCODE RetCode = SQL_SUCCESS; + int iCount = 0; + + // Allocate memory for each column + + AllocateBindings(hStmt, cCols, &pFirstBinding, &cDisplaySize); + + // Set the display mode and write the titles + + DisplayTitles(hStmt, cDisplaySize+1, pFirstBinding); + + + // Fetch and display the data + + bool fNoData = false; + + do { + // Fetch a row + + if (iCount++ >= gHeight - 2) + { + int nInputChar; + bool fEnterReceived = false; + + while(!fEnterReceived) + { + wprintf(L" "); + SetConsole(cDisplaySize+2, TRUE); + wprintf(L" Press ENTER to continue, Q to quit (height:%hd)", gHeight); + SetConsole(cDisplaySize+2, FALSE); + + nInputChar = _getch(); + wprintf(L"\n"); + if ((nInputChar == 'Q') || (nInputChar == 'q')) + { + goto Exit; + } + else if ('\r' == nInputChar) + { + fEnterReceived = true; + } + // else loop back to display prompt again + } + + iCount = 1; + DisplayTitles(hStmt, cDisplaySize+1, pFirstBinding); + } + + TRYODBC(hStmt, SQL_HANDLE_STMT, RetCode = SQLFetch(hStmt)); + + if (RetCode == SQL_NO_DATA_FOUND) + { + fNoData = true; + } + else + { + + // Display the data. Ignore truncations + + for (pThisBinding = pFirstBinding; + pThisBinding; + pThisBinding = pThisBinding->sNext) + { + if (pThisBinding->indPtr != SQL_NULL_DATA) + { + wprintf(pThisBinding->fChar ? DISPLAY_FORMAT_C:DISPLAY_FORMAT, + PIPE, + pThisBinding->cDisplaySize, + pThisBinding->cDisplaySize, + pThisBinding->wszBuffer); + } + else + { + wprintf(DISPLAY_FORMAT_C, + PIPE, + pThisBinding->cDisplaySize, + pThisBinding->cDisplaySize, + L""); + } + } + wprintf(L" %c\n",PIPE); + } + } while (!fNoData); + + SetConsole(cDisplaySize+2, TRUE); + wprintf(L"%*.*s", cDisplaySize+2, cDisplaySize+2, L" "); + SetConsole(cDisplaySize+2, FALSE); + wprintf(L"\n"); + +Exit: + // Clean up the allocated buffers + + while (pFirstBinding) + { + pThisBinding = pFirstBinding->sNext; + free(pFirstBinding->wszBuffer); + free(pFirstBinding); + pFirstBinding = pThisBinding; + } +} + +/************************************************************************ +/* AllocateBindings: Get column information and allocate bindings +/* for each column. +/* +/* Parameters: +/* hStmt Statement handle +/* cCols Number of columns in the result set +/* *lppBinding Binding pointer (returned) +/* lpDisplay Display size of one line +/************************************************************************/ + +void AllocateBindings(HSTMT hStmt, + SQLSMALLINT cCols, + BINDING **ppBinding, + SQLSMALLINT *pDisplay) +{ + SQLSMALLINT iCol; + BINDING *pThisBinding, *pLastBinding = NULL; + SQLLEN cchDisplay, ssType; + SQLSMALLINT cchColumnNameLength; + + *pDisplay = 0; + + for (iCol = 1; iCol <= cCols; iCol++) + { + pThisBinding = (BINDING *)(malloc(sizeof(BINDING))); + if (!(pThisBinding)) + { + fwprintf(stderr, L"Out of memory!\n"); + exit(-100); + } + + if (iCol == 1) + { + *ppBinding = pThisBinding; + } + else + { + pLastBinding->sNext = pThisBinding; + } + pLastBinding = pThisBinding; + + + // Figure out the display length of the column (we will + // bind to char since we are only displaying data, in general + // you should bind to the appropriate C type if you are going + // to manipulate data since it is much faster...) + + TRYODBC(hStmt, + SQL_HANDLE_STMT, + SQLColAttribute(hStmt, + iCol, + SQL_DESC_DISPLAY_SIZE, + NULL, + 0, + NULL, + &cchDisplay)); + + + // Figure out if this is a character or numeric column; this is + // used to determine if we want to display the data left- or right- + // aligned. + + // SQL_DESC_CONCISE_TYPE maps to the 1.x SQL_COLUMN_TYPE. + // This is what you must use if you want to work + // against a 2.x driver. + + TRYODBC(hStmt, + SQL_HANDLE_STMT, + SQLColAttribute(hStmt, + iCol, + SQL_DESC_CONCISE_TYPE, + NULL, + 0, + NULL, + &ssType)); + + pThisBinding->fChar = (ssType == SQL_CHAR || + ssType == SQL_VARCHAR || + ssType == SQL_LONGVARCHAR); + + pThisBinding->sNext = NULL; + + // Arbitrary limit on display size + if (cchDisplay > DISPLAY_MAX) + cchDisplay = DISPLAY_MAX; + + // Allocate a buffer big enough to hold the text representation + // of the data. Add one character for the null terminator + + pThisBinding->wszBuffer = (WCHAR *)malloc((cchDisplay+1) * sizeof(WCHAR)); + + if (!(pThisBinding->wszBuffer)) + { + fwprintf(stderr, L"Out of memory!\n"); + exit(-100); + } + + // Map this buffer to the driver's buffer. At Fetch time, + // the driver will fill in this data. Note that the size is + // count of bytes (for Unicode). All ODBC functions that take + // SQLPOINTER use count of bytes; all functions that take only + // strings use count of characters. + + TRYODBC(hStmt, + SQL_HANDLE_STMT, + SQLBindCol(hStmt, + iCol, + SQL_C_TCHAR, + (SQLPOINTER) pThisBinding->wszBuffer, + (cchDisplay + 1) * sizeof(WCHAR), + &pThisBinding->indPtr)); + + + // Now set the display size that we will use to display + // the data. Figure out the length of the column name + + TRYODBC(hStmt, + SQL_HANDLE_STMT, + SQLColAttribute(hStmt, + iCol, + SQL_DESC_NAME, + NULL, + 0, + &cchColumnNameLength, + NULL)); + + pThisBinding->cDisplaySize = max((SQLSMALLINT)cchDisplay, cchColumnNameLength); + if (pThisBinding->cDisplaySize < NULL_SIZE) + pThisBinding->cDisplaySize = NULL_SIZE; + + *pDisplay += pThisBinding->cDisplaySize + DISPLAY_FORMAT_EXTRA; + + } + + return; + +Exit: + + exit(-1); + + return; +} + + +/************************************************************************ +/* DisplayTitles: print the titles of all the columns and set the +/* shell window's width +/* +/* Parameters: +/* hStmt Statement handle +/* cDisplaySize Total display size +/* pBinding list of binding information +/************************************************************************/ + +void DisplayTitles(HSTMT hStmt, + DWORD cDisplaySize, + BINDING *pBinding) +{ + WCHAR wszTitle[DISPLAY_MAX]; + SQLSMALLINT iCol = 1; + + SetConsole(cDisplaySize+2, TRUE); + + for (; pBinding; pBinding = pBinding->sNext) + { + TRYODBC(hStmt, + SQL_HANDLE_STMT, + SQLColAttribute(hStmt, + iCol++, + SQL_DESC_NAME, + wszTitle, + sizeof(wszTitle), // Note count of bytes! + NULL, + NULL)); + + wprintf(DISPLAY_FORMAT_C, + PIPE, + pBinding->cDisplaySize, + pBinding->cDisplaySize, + wszTitle); + } + +Exit: + + wprintf(L" %c", PIPE); + SetConsole(cDisplaySize+2, FALSE); + wprintf(L"\n"); + +} + + +/************************************************************************ +/* SetConsole: sets console display size and video mode +/* +/* Parameters +/* siDisplaySize Console display size +/* fInvert Invert video? +/************************************************************************/ + +void SetConsole(DWORD dwDisplaySize, + BOOL fInvert) +{ + HANDLE hConsole; + CONSOLE_SCREEN_BUFFER_INFO csbInfo; + + // Reset the console screen buffer size if necessary + + hConsole = GetStdHandle(STD_OUTPUT_HANDLE); + + if (hConsole != INVALID_HANDLE_VALUE) + { + if (GetConsoleScreenBufferInfo(hConsole, &csbInfo)) + { + if (csbInfo.dwSize.X < (SHORT) dwDisplaySize) + { + csbInfo.dwSize.X = (SHORT) dwDisplaySize; + SetConsoleScreenBufferSize(hConsole, csbInfo.dwSize); + } + + gHeight = csbInfo.dwSize.Y; + } + + if (fInvert) + { + SetConsoleTextAttribute(hConsole, (WORD)(csbInfo.wAttributes | BACKGROUND_BLUE)); + } + else + { + SetConsoleTextAttribute(hConsole, (WORD)(csbInfo.wAttributes & ~(BACKGROUND_BLUE))); + } + } +} + + +/************************************************************************ +/* HandleDiagnosticRecord : display error/warning information +/* +/* Parameters: +/* hHandle ODBC handle +/* hType Type of handle (HANDLE_STMT, HANDLE_ENV, HANDLE_DBC) +/* RetCode Return code of failing command +/************************************************************************/ + +void HandleDiagnosticRecord (SQLHANDLE hHandle, + SQLSMALLINT hType, + RETCODE RetCode) +{ + SQLSMALLINT iRec = 0; + SQLINTEGER iError; + WCHAR wszMessage[1000]; + WCHAR wszState[SQL_SQLSTATE_SIZE+1]; + + + if (RetCode == SQL_INVALID_HANDLE) + { + fwprintf(stderr, L"Invalid handle!\n"); + return; + } + + while (SQLGetDiagRec(hType, + hHandle, + ++iRec, + wszState, + &iError, + wszMessage, + (SQLSMALLINT)(sizeof(wszMessage) / sizeof(WCHAR)), + (SQLSMALLINT *)NULL) == SQL_SUCCESS) + { + // Hide data truncated.. + if (wcsncmp(wszState, L"01004", 5)) + { + fwprintf(stderr, L"[%5.5s] %s (%d)\n", wszState, wszMessage, iError); + } + } +} diff --git a/src/connector/odbc/examples/go/odbc.go b/src/connector/odbc/examples/go/odbc.go new file mode 100644 index 0000000000000000000000000000000000000000..4d9c760c4e87a4a899051edc74692ecca8a19d15 --- /dev/null +++ b/src/connector/odbc/examples/go/odbc.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "database/sql" + "flag" + "log" + "os" + "os/signal" + "time" + _ "github.com/alexbrainman/odbc" +) + +var pool *sql.DB // Database connection pool. + +func main() { + id := flag.Int64("id", 32768, "person ID to find") + dsn := flag.String("dsn", os.Getenv("DSN"), "connection data source name") + flag.Parse() + + if len(*dsn) == 0 { + log.Fatal("missing dsn flag") + } + if *id == 0 { + log.Fatal("missing person ID") + } + var err error + + // Opening a driver typically will not attempt to connect to the database. + pool, err = sql.Open("odbc", *dsn) + if err != nil { + // This will not be a connection error, but a DSN parse error or + // another initialization error. + log.Fatal("unable to use data source name", err) + } + defer pool.Close() + + pool.SetConnMaxLifetime(0) + pool.SetMaxIdleConns(3) + pool.SetMaxOpenConns(3) + + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + appSignal := make(chan os.Signal, 3) + signal.Notify(appSignal, os.Interrupt) + + go func() { + select { + case <-appSignal: + stop() + } + }() + + Ping(ctx) + + Query(ctx, *id) +} + +// Ping the database to verify DSN provided by the user is valid and the +// server accessible. If the ping fails exit the program with an error. +func Ping(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + if err := pool.PingContext(ctx); err != nil { + log.Fatalf("unable to connect to database: %v", err) + } +} + +// Query the database for the information requested and prints the results. +// If the query fails exit the program with an error. +func Query(ctx context.Context, id int64) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + var name string + err := pool.QueryRowContext(ctx, "select name from m.t").Scan(&name) + if err != nil { + log.Fatal("unable to execute search query", err) + } + log.Println("name=", name) +} + diff --git a/src/connector/odbc/examples/js/.gitignore b/src/connector/odbc/examples/js/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/connector/odbc/examples/js/odbc.js b/src/connector/odbc/examples/js/odbc.js new file mode 100755 index 0000000000000000000000000000000000000000..78a4365c11ddec66666bc54755c9429c34f53af3 --- /dev/null +++ b/src/connector/odbc/examples/js/odbc.js @@ -0,0 +1,122 @@ +#!/usr/bin/env node + +const odbc = require('odbc'); +const path = require('path'); + +function usage() { + var arg = path.basename(process.argv[1]); + console.error(`usage:`); + console.error(`${arg} --DSN --UID --PWD --Server `); + console.error(`${arg} -C `); + console.error(` conn_str eg: 'DSN={TAOS_DSN};UID=root;PWD=taosdata;Server=host:port'`); +} + +var cfg = { }; + +if (process.argv.length==2) { + usage(); + process.exit(0); +} + +var i; +for (i=2; i=process.argv.length) { + console.error(`expecting after --DSN but got nothing`); + usage(process.argv[1]); + process.exit(1); + } + arg = process.argv[i]; + cfg.dsn = arg; + continue; + } + if (arg=="--UID") { + ++i; + if (i>=process.argv.length) { + console.error(`expecting after --UID but got nothing`); + usage(process.argv[1]); + process.exit(1); + } + arg = process.argv[i]; + cfg.uid = arg; + continue; + } + if (arg=="--PWD") { + ++i; + if (i>=process.argv.length) { + console.error(`expecting after --PWD but got nothing`); + usage(process.argv[1]); + process.exit(1); + } + arg = process.argv[i]; + cfg.pwd = arg; + continue; + } + if (arg=="--Server") { + ++i; + if (i>=process.argv.length) { + console.error(`expecting after --Server but got nothing`); + usage(process.argv[1]); + process.exit(1); + } + arg = process.argv[i]; + cfg.server = arg; + continue; + } + if (arg=="-C") { + ++i; + if (i>=process.argv.length) { + console.error(`expecting after -C but got nothing`); + console.error(` conn_str eg: 'DSN={TAOS_DSN};UID=root;PWD=taosdata;Server=host:port'`); + process.exit(1); + } + arg = process.argv[i]; + cfg.conn_str = arg; + continue; + } + console.error(`unknown argument: [${arg}]`); + usage(process.argv[1]); + process.exit(1); +} + +var connectionString = cfg.conn_str; + +if (!cfg.conn_str) { + connectionString = `DSN={${cfg.dsn}}; UID=${cfg.uid}; PWD=${cfg.pwd}; Server=${cfg.server}`; +} + +(async function () { + const connStr = connectionString; + try { + console.log(`connecting [${connStr}]...`); + const connection = await odbc.connect(connStr); + await connection.query('create database if not exists m'); + await connection.query('use m'); + await connection.query('drop table if exists t'); + await connection.query('create table t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(10), name nchar(3))'); + await connection.query('insert into t values("2020-01-02 12:34:56.781", 1, 127, 32767, 32768, 32769, 123.456, 789.987, "hello", "我和你")'); + console.log('.........'); + result = await connection.query('select * from t'); + console.log(result[0]); + + + statement = await connection.createStatement(); + await statement.prepare('INSERT INTO t (ts, v1) VALUES(?, ?)'); + await statement.bind(['2020-02-02 11:22:33.449', 89]); + result = await statement.execute(); + console.log(result); + + result = await connection.query('select * from t'); + console.log(result[0]); + console.log(result[1]); + } catch (e) { + console.log('error:', e); + } +})(); + diff --git a/src/connector/odbc/examples/js/package.json b/src/connector/odbc/examples/js/package.json new file mode 100644 index 0000000000000000000000000000000000000000..28a04dc32f0d1fc9f1c06de174241b10606f8639 --- /dev/null +++ b/src/connector/odbc/examples/js/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "odbc": "^2.3.6" + } +} diff --git a/src/connector/odbc/examples/lua/odbc.lua b/src/connector/odbc/examples/lua/odbc.lua new file mode 100644 index 0000000000000000000000000000000000000000..e3388358cdab53d7513650192c6a272fadd0e185 --- /dev/null +++ b/src/connector/odbc/examples/lua/odbc.lua @@ -0,0 +1,48 @@ +package.cpath = package.cpath .. ";/usr/local/lib/lib?.dylib" +-- load driver +local driver = require "luasql.odbc" +-- create environment object +env = assert (driver.odbc()) +-- connect to data source +con = assert (env:connect("TAOS_DSN", "root", "taosdata")) +-- reset our table +-- res = con:execute"DROP TABLE people" +-- res = assert (con:execute[[ +-- CREATE TABLE people( +-- name varchar(50), +-- email varchar(50) +-- ) +-- ]]) +-- -- add a few elements +-- list = { +-- { name="Jose das Couves", email="jose@couves.com", }, +-- { name="Manoel Joaquim", email="manoel.joaquim@cafundo.com", }, +-- { name="Maria das Dores", email="maria@dores.com", }, +-- } +-- for i, p in pairs (list) do +-- res = assert (con:execute(string.format([[ +-- INSERT INTO people +-- VALUES ('%s', '%s')]], p.name, p.email) +-- )) +-- end +-- -- retrieve a cursor +-- cur = assert (con:execute"SELECT name, email from people") +-- -- print all rows, the rows will be indexed by field names +-- row = cur:fetch ({}, "a") +-- while row do +-- print(string.format("Name: %s, E-mail: %s", row.name, row.email)) +-- -- reusing the table of results +-- row = cur:fetch (row, "a") +-- end +cur = assert(con:execute"select * from m.t") +row = cur:fetch({}, "a") +while row do + print(string.format("Name: %s", row.name)) + row = cur:fetch(row, "a") +end + +-- close everything +cur:close() -- already closed because all the result set was consumed +con:close() +env:close() + diff --git a/src/connector/odbc/examples/py/odbc.py b/src/connector/odbc/examples/py/odbc.py new file mode 100644 index 0000000000000000000000000000000000000000..e6a4bc73aef3e19bc56e817325acd62d21156d67 --- /dev/null +++ b/src/connector/odbc/examples/py/odbc.py @@ -0,0 +1,111 @@ +import pyodbc +import argparse +import sys + +parser = argparse.ArgumentParser(description='Access TDengine via ODBC.') +parser.add_argument('--DSN', help='DSN to use') +parser.add_argument('--UID', help='UID to use') +parser.add_argument('--PWD', help='PWD to use') +parser.add_argument('--Server', help='Server to use') +parser.add_argument('-C', metavar='CONNSTR', help='Connection string to use') + +args = parser.parse_args() + +a = 'DSN=%s'%args.DSN if args.DSN else None +b = 'UID=%s'%args.UID if args.UID else None +c = 'PWD=%s'%args.PWD if args.PWD else None +d = 'Server=%s'%args.Server if args.Server else None +conn_str = ';'.join(filter(None, [a,b,c,d])) if args.DSN else None +conn_str = conn_str if conn_str else args.C +if not conn_str: + parser.print_help(file=sys.stderr) + exit() + +print('connecting: [%s]' % conn_str) +cnxn = pyodbc.connect(conn_str, autocommit=True) +cnxn.setdecoding(pyodbc.SQL_CHAR, encoding='utf-8') + +cursor = cnxn.cursor() +cursor.execute("drop database if exists db"); +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("create database db"); +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("create table db.mt (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(10), blob nchar(10))"); +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("insert into db.mt values('2020-10-13 06:44:00.123', 1, 127, 32767, 2147483647, 32769, 123.456, 789.987, 'hello', 'helloworld')") +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("insert into db.mt values(?,?,?,?,?,?,?,?,?,?)", "2020-10-13 07:06:00.234", 0, 127, 32767, 32768, 32769, 123.456, 789.987, "hel后lo".encode('utf-8'), "wo哈rlxd129") +##cursor.execute("insert into db.mt values(?,?,?,?,?,?,?,?,?,?)", 1502535178128, 9223372036854775807, 127, 32767, 32768, 32769, 123.456, 789.987, "hel后lo".encode('utf-8'), "wo哈rlxd123"); +cursor.close() + +cursor = cnxn.cursor() +cursor.execute(""" +INSERT INTO db.mt (ts,b,v1,v2,v4,v8,f4,f8,bin,blob) values (?,?,?,?,?,?,?,?,?,?) +""", +"2020-12-12 00:00:00", +'true', +'-127', +'-32767', +'-2147483647', +'-9223372036854775807', +'-1.23e10', +'-11.23e6', +'abcdefghij'.encode('utf-8'), +"人啊大发测试及abc") +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("drop database if exists db"); +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("create database db"); +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("create table db.t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(4), blob nchar(4))"); +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("insert into db.t values('2020-10-13 06:44:00', 1, 127, 32767, 32768, 32769, 123.456, 789.987, 'hell', 'w我你z')") +cursor.close() + +cursor = cnxn.cursor() +cursor.execute("create table db.v (ts timestamp, v1 tinyint, v2 smallint, name nchar(10), ts2 timestamp)") +cursor.close() + +params = [ ('2020-10-16 00:00:00.123', 19, '2111-01-02 01:02:03.123'), + ('2020-10-16 00:00:01', 41, '2111-01-02 01:02:03.423'), + ('2020-10-16 00:00:02', 57, '2111-01-02 01:02:03.153'), + ('2020-10-16 00:00:03.009', 26, '2111-01-02 01:02:03.623') ] +cursor = cnxn.cursor() +cursor.fast_executemany = True +print('py:...................') +cursor.executemany("insert into db.v (ts, v1, ts2) values (?, ?, ?)", params) +print('py:...................') +cursor.close() + +## cursor = cnxn.cursor() +## cursor.execute("SELECT * from db.v where v1 > ?", 4) +## row = cursor.fetchone() +## while row: +## print(row) +## row = cursor.fetchone() +## cursor.close() +## +## cursor = cnxn.cursor() +## cursor.execute("SELECT * from db.v where v1 > ?", '5') +## row = cursor.fetchone() +## while row: +## print(row) +## row = cursor.fetchone() +## cursor.close() + diff --git a/src/connector/odbc/examples/rust/.gitignore b/src/connector/odbc/examples/rust/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..03314f77b5aa454994bf08c40943f4012964eb0a --- /dev/null +++ b/src/connector/odbc/examples/rust/.gitignore @@ -0,0 +1 @@ +Cargo.lock diff --git a/src/connector/odbc/examples/rust/main/Cargo.toml b/src/connector/odbc/examples/rust/main/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..171dcf52b2bfcf3bb56fa49c4e228871311e90aa --- /dev/null +++ b/src/connector/odbc/examples/rust/main/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "main" +version = "0.1.0" +authors = ["freemine "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +odbc = "0.17.0" +env_logger = "0.8.2" + diff --git a/src/connector/odbc/examples/rust/main/src/main.rs b/src/connector/odbc/examples/rust/main/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..112f0d309b51a0a75a7e580f764c697a2215b9da --- /dev/null +++ b/src/connector/odbc/examples/rust/main/src/main.rs @@ -0,0 +1,49 @@ +extern crate odbc; +// Use this crate and set environmet variable RUST_LOG=odbc to see ODBC warnings +extern crate env_logger; +use odbc::*; +use odbc_safe::AutocommitOn; +use std::env; + +fn main() { + + env_logger::init(); + + let conn_str = env::var("DSN").unwrap(); + match connect(&conn_str) { + Ok(()) => println!("Success"), + Err(diag) => println!("Error: {}", diag), + } +} + +fn connect(conn_str: &str) -> std::result::Result<(), DiagnosticRecord> { + + let env = create_environment_v3().map_err(|e| e.unwrap())?; + + let conn = env.connect_with_connection_string(conn_str)?; + execute_statement(&conn) +} + +fn execute_statement<'env>(conn: &Connection<'env, AutocommitOn>) -> Result<()> { + let stmt = Statement::with_parent(conn)?; + + match stmt.exec_direct("select * from m.t")? { + Data(mut stmt) => { + let cols = stmt.num_result_cols()?; + println!("cols: {}", cols); + while let Some(mut cursor) = stmt.fetch()? { + for i in 1..(cols + 1) { + match cursor.get_data::<&str>(i as u16)? { + Some(val) => print!(" {}", val), + None => print!(" NULL"), + } + } + println!(""); + } + } + NoData(_) => println!("Query executed, no data returned"), + } + + Ok(()) +} + diff --git a/src/connector/odbc/tests/create_data.stmts b/src/connector/odbc/samples/create_data.stmts similarity index 51% rename from src/connector/odbc/tests/create_data.stmts rename to src/connector/odbc/samples/create_data.stmts index 549cb583d8322906b4bdaffafde8eb510cb91c90..3d41c0db6447ed8a45390c2a50d73697992dbf52 100644 --- a/src/connector/odbc/tests/create_data.stmts +++ b/src/connector/odbc/samples/create_data.stmts @@ -1,12 +1,17 @@ +#P: positive sample +#N: negative sample + P:drop database if exists m; P:create database m; P:use m; P:drop table if exists t; -P:create table t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, blob binary(3), name nchar(1)); -P:insert into t (ts, blob, name) values('2020-10-10 00:00:00', 0, 1); -P:insert into t (ts, blob, name) values('2020-10-10 00:00:00.001', 1, 2); +P:create table t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, blob binary(3), name nchar(1), ts2 nchar(148)); +#P:insert into t (ts, blob, name) values('2020-10-10 00:00:00', 0, 1); +#P:insert into t (ts, blob, name) values('2020-10-10 00:00:00.001', 1, 2); P:insert into t (ts, blob, name) values('2020-10-10 00:00:00.002', '你', '好'); P:insert into t (ts, blob, name) values('2020-10-10 00:00:00.003', 'abc', 'd'); P:select * from t; +P:create table v (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, blob binary(3), name nchar(1), ts2 nchar(23)); + diff --git a/src/connector/odbc/tests/query_data.stmts b/src/connector/odbc/samples/query_data.stmts similarity index 100% rename from src/connector/odbc/tests/query_data.stmts rename to src/connector/odbc/samples/query_data.stmts diff --git a/src/connector/odbc/tests/select.stmts b/src/connector/odbc/samples/select.stmts similarity index 100% rename from src/connector/odbc/tests/select.stmts rename to src/connector/odbc/samples/select.stmts diff --git a/src/connector/odbc/tests/simples.stmts b/src/connector/odbc/samples/simples.stmts similarity index 100% rename from src/connector/odbc/tests/simples.stmts rename to src/connector/odbc/samples/simples.stmts diff --git a/src/connector/odbc/src/CMakeLists.txt b/src/connector/odbc/src/CMakeLists.txt index 67357cb4698b2885b563fd54133f36aace38c54b..f0e50415e2e4f14e1c247b834e1e52a2c2fd2868 100644 --- a/src/connector/odbc/src/CMakeLists.txt +++ b/src/connector/odbc/src/CMakeLists.txt @@ -1,6 +1,8 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +add_subdirectory(base) + IF (TD_LINUX_64) FLEX_TARGET(todbcFlexScanner todbc_scanner.l @@ -15,12 +17,35 @@ IF (TD_LINUX_64) ADD_LIBRARY(todbc SHARED ${SRC} ${todbc_flex_scanner_src}) SET_TARGET_PROPERTIES(todbc PROPERTIES CLEAN_DIRECT_OUTPUT 1) SET_TARGET_PROPERTIES(todbc PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) - TARGET_LINK_LIBRARIES(todbc taos odbcinst) + TARGET_LINK_LIBRARIES(todbc todbc_base taos odbcinst) target_include_directories(todbc PUBLIC .) install(CODE "execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/install.sh ${CMAKE_BINARY_DIR})") ENDIF () +IF (TD_DARWIN) + FLEX_TARGET(todbcFlexScanner + todbc_scanner.l + ${CMAKE_CURRENT_BINARY_DIR}/todbc_scanner.c + ) + set(todbc_flex_scanner_src + ${FLEX_todbcFlexScanner_OUTPUTS} + ) + set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/todbc_scanner.c PROPERTIES COMPILE_OPTIONS "-Wno-conversion") + AUX_SOURCE_DIRECTORY(. SRC) + + # generate dynamic library (*.dylib) + ADD_LIBRARY(todbc SHARED ${SRC} ${todbc_flex_scanner_src}) + SET_TARGET_PROPERTIES(todbc PROPERTIES CLEAN_DIRECT_OUTPUT 1) + SET_TARGET_PROPERTIES(todbc PROPERTIES VERSION ${TD_VER_NUMBER} SOVERSION 1) + TARGET_LINK_LIBRARIES(todbc todbc_base taos odbcinst) + target_include_directories(todbc PUBLIC .) + target_include_directories(todbc PRIVATE /usr/local/include) + target_link_directories(todbc PUBLIC /usr/local/lib) + + install(CODE "execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/install.sh ${CMAKE_BINARY_DIR})") +ENDIF () + IF (TD_WINDOWS_64) FLEX_TARGET(todbcFlexScanner todbc_scanner.l @@ -37,7 +62,7 @@ IF (TD_WINDOWS_64) ${todbc_flex_scanner_src} ${CMAKE_CURRENT_BINARY_DIR}/todbc.rc todbc.def) - TARGET_LINK_LIBRARIES(todbc taos_static odbccp32 legacy_stdio_definitions) + TARGET_LINK_LIBRARIES(todbc todbc_base taos_static odbccp32 legacy_stdio_definitions) target_include_directories(todbc PUBLIC .) target_compile_definitions(todbc PRIVATE "todbc_EXPORT") @@ -52,3 +77,4 @@ IF (TD_WINDOWS_64) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/todbc.exp DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/todbc.dll DESTINATION driver) ENDIF () + diff --git a/src/connector/odbc/src/base.c b/src/connector/odbc/src/base.c new file mode 100644 index 0000000000000000000000000000000000000000..54b55b48edcc63c444e05e2b3a234ac98025506e --- /dev/null +++ b/src/connector/odbc/src/base.c @@ -0,0 +1,2713 @@ +#include "base.h" + +#include "base/env.h" +#include "base/null_conn.h" +#include "base/tsdb_impl.h" + +#include "todbc_flex.h" +#include "todbc_tls.h" +#include "todbc_util.h" + +#ifdef _MSC_VER +#include +#endif // _MSC_VER + +#define PROFILING 0 +#define LOGGING 0 + +#define PROFILE(r_0911, statement) \ +do { \ + if (!PROFILING) { \ + if (LOGGING) D(""); \ + statement; \ + if (LOGGING) D("r=%zx", (size_t)r_0911); \ + break; \ + } \ + if (LOGGING) D(""); \ + struct timeval tv0, tv1; \ + gettimeofday(&tv0, NULL); \ + statement; \ + gettimeofday(&tv1, NULL); \ + double delta = difftime(tv1.tv_sec, tv0.tv_sec); \ + delta *= 1000000; \ + delta += (double)(tv1.tv_usec-tv0.tv_usec); \ + delta /= 1000000; \ + D("%s: elapsed: [%.6f]s", #statement, delta); \ + if (LOGGING) D("r=%zx", (size_t)r_0911); \ +} while (0) + +#define P(fmt,...) do { \ + if (LOGGING) { \ + D(fmt, ##__VA_ARGS__); \ + } \ +} while (0) + + + +static SQLRETURN doSQLAllocConnect(SQLHENV EnvironmentHandle, SQLHDBC *ConnectionHandle); + +SQLRETURN SQL_API SQLAllocConnect(SQLHENV EnvironmentHandle, SQLHDBC *ConnectionHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_ENV, EnvironmentHandle); + PROFILE(r, r = doSQLAllocConnect(EnvironmentHandle, ConnectionHandle)); + todbc_tls_buf_reclaim(); + return r; +} + + +static SQLRETURN doSQLAllocEnv(SQLHENV *EnvironmentHandle); + +SQLRETURN SQL_API SQLAllocEnv(SQLHENV *EnvironmentHandle) +{ + SQLRETURN r; + PROFILE(r, r = doSQLAllocEnv(EnvironmentHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLAllocHandle(SQLSMALLINT HandleType, SQLHANDLE InputHandle, SQLHANDLE *OutputHandle); + +SQLRETURN SQL_API SQLAllocHandle(SQLSMALLINT HandleType, SQLHANDLE InputHandle, SQLHANDLE *OutputHandle) +{ + SQLRETURN r; + // HandleType is NOT the type of InputHandle + PROFILE(r, r = doSQLAllocHandle(HandleType, InputHandle, OutputHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLAllocStmt(SQLHDBC ConnectionHandle, SQLHSTMT *StatementHandle); + +SQLRETURN SQL_API SQLAllocStmt(SQLHDBC ConnectionHandle, SQLHSTMT *StatementHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = doSQLAllocStmt(ConnectionHandle, StatementHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLBindCol(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, + SQLPOINTER TargetValue, + SQLLEN BufferLength, SQLLEN *StrLen_or_IndPtr); + +SQLRETURN SQL_API SQLBindCol(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, + SQLPOINTER TargetValue, + SQLLEN BufferLength, SQLLEN *StrLen_or_IndPtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLBindCol(StatementHandle, ColumnNumber, TargetType, + TargetValue, BufferLength, StrLen_or_IndPtr)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLBindParam(SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, SQLSMALLINT ValueType, + SQLSMALLINT ParameterType, SQLULEN LengthPrecision, + SQLSMALLINT ParameterScale, SQLPOINTER ParameterValue, + SQLLEN *StrLen_or_IndPtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLCancel(SQLHSTMT StatementHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLCancelHandle(SQLSMALLINT HandleType, SQLHANDLE InputHandle) +{ + SQLRETURN r; + errs_clear(HandleType, InputHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLCloseCursor(SQLHSTMT StatementHandle); + +SQLRETURN SQL_API SQLCloseCursor(SQLHSTMT StatementHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLCloseCursor(StatementHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLColAttribute (SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLUSMALLINT FieldIdentifier, + SQLPOINTER CharacterAttribute, SQLSMALLINT BufferLength, + SQLSMALLINT *StringLength, SQLLEN *NumericAttribute) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLColumns(SQLHSTMT StatementHandle, + SQLCHAR *CatalogName, SQLSMALLINT NameLength1, + SQLCHAR *SchemaName, SQLSMALLINT NameLength2, + SQLCHAR *TableName, SQLSMALLINT NameLength3, + SQLCHAR *ColumnName, SQLSMALLINT NameLength4) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLCompleteAsync(SQLSMALLINT HandleType, SQLHANDLE Handle, RETCODE* AsyncRetCodePtr) +{ + SQLRETURN r; + errs_clear(HandleType, Handle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLConnect(SQLHDBC ConnectionHandle, + SQLCHAR *ServerName, SQLSMALLINT NameLength1, + SQLCHAR *UserName, SQLSMALLINT NameLength2, + SQLCHAR *Authentication, SQLSMALLINT NameLength3); + +SQLRETURN SQL_API SQLConnect(SQLHDBC ConnectionHandle, + SQLCHAR *ServerName, SQLSMALLINT NameLength1, + SQLCHAR *UserName, SQLSMALLINT NameLength2, + SQLCHAR *Authentication, SQLSMALLINT NameLength3) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = doSQLConnect(ConnectionHandle, ServerName, NameLength1, UserName, NameLength2, Authentication, NameLength3)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLCopyDesc(SQLHDESC SourceDescHandle, SQLHDESC TargetDescHandle) +{ + SQLRETURN r; + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLDataSources(SQLHENV EnvironmentHandle, + SQLUSMALLINT Direction, SQLCHAR *ServerName, SQLSMALLINT BufferLength1, SQLSMALLINT *NameLength1Ptr, + SQLCHAR *Description, SQLSMALLINT BufferLength2, + SQLSMALLINT *NameLength2Ptr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_ENV, EnvironmentHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLDescribeCol(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLCHAR *ColumnName, SQLSMALLINT BufferLength, + SQLSMALLINT *NameLength, + SQLSMALLINT *DataType, SQLULEN *ColumnSize, + SQLSMALLINT *DecimalDigits, SQLSMALLINT *Nullable); + +SQLRETURN SQL_API SQLDescribeCol(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLCHAR *ColumnName, SQLSMALLINT BufferLength, + SQLSMALLINT *NameLength, + SQLSMALLINT *DataType, SQLULEN *ColumnSize, + SQLSMALLINT *DecimalDigits, SQLSMALLINT *Nullable) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLDescribeCol(StatementHandle, ColumnNumber, ColumnName, BufferLength, + NameLength, DataType, ColumnSize, DecimalDigits, Nullable)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLDisconnect(SQLHDBC ConnectionHandle); + +SQLRETURN SQL_API SQLDisconnect(SQLHDBC ConnectionHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = doSQLDisconnect(ConnectionHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLEndTran(SQLSMALLINT HandleType, SQLHANDLE Handle, SQLSMALLINT CompletionType) +{ + SQLRETURN r; + errs_clear(HandleType, Handle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLError(SQLHENV EnvironmentHandle, + SQLHDBC ConnectionHandle, SQLHSTMT StatementHandle, + SQLCHAR *Sqlstate, SQLINTEGER *NativeError, + SQLCHAR *MessageText, SQLSMALLINT BufferLength, + SQLSMALLINT *TextLength); + +SQLRETURN SQL_API not_support_SQLError(SQLHENV EnvironmentHandle, + SQLHDBC ConnectionHandle, SQLHSTMT StatementHandle, + SQLCHAR *Sqlstate, SQLINTEGER *NativeError, + SQLCHAR *MessageText, SQLSMALLINT BufferLength, + SQLSMALLINT *TextLength) +{ + SQLRETURN r; + PROFILE(r, r = doSQLError(EnvironmentHandle, ConnectionHandle, StatementHandle, + Sqlstate, NativeError, MessageText, BufferLength, TextLength)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLExecDirect(SQLHSTMT StatementHandle, SQLCHAR* StatementText, SQLINTEGER TextLength); + +SQLRETURN SQL_API SQLExecDirect(SQLHSTMT StatementHandle, SQLCHAR* StatementText, SQLINTEGER TextLength) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLExecDirect(StatementHandle, StatementText, TextLength)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLExecute(SQLHSTMT StatementHandle); + +SQLRETURN SQL_API SQLExecute(SQLHSTMT StatementHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLExecute(StatementHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLFetch(SQLHSTMT StatementHandle); + +SQLRETURN SQL_API SQLFetch(SQLHSTMT StatementHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLFetch(StatementHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLFetchScroll(SQLHSTMT StatementHandle, SQLSMALLINT FetchOrientation, SQLLEN FetchOffset) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLFreeConnect(SQLHDBC ConnectionHandle); + +SQLRETURN SQL_API SQLFreeConnect(SQLHDBC ConnectionHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = doSQLFreeConnect(ConnectionHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLFreeEnv(SQLHENV EnvironmentHandle); + +SQLRETURN SQL_API SQLFreeEnv(SQLHENV EnvironmentHandle) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_ENV, EnvironmentHandle); + PROFILE(r, r = doSQLFreeEnv(EnvironmentHandle)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLFreeHandle(SQLSMALLINT HandleType, SQLHANDLE Handle); + +SQLRETURN SQL_API SQLFreeHandle(SQLSMALLINT HandleType, SQLHANDLE Handle) +{ + SQLRETURN r; + errs_clear(HandleType, Handle); + PROFILE(r, r = doSQLFreeHandle(HandleType, Handle)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLFreeStmt(SQLHSTMT StatementHandle, SQLUSMALLINT Option); + +SQLRETURN SQL_API SQLFreeStmt(SQLHSTMT StatementHandle, SQLUSMALLINT Option) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLFreeStmt(StatementHandle, Option)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLGetConnectAttr(SQLHDBC ConnectionHandle, + SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLengthPtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLGetConnectOption(SQLHDBC ConnectionHandle, SQLUSMALLINT Option, SQLPOINTER Value) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLGetCursorName(SQLHSTMT StatementHandle, SQLCHAR *CursorName, SQLSMALLINT BufferLength, SQLSMALLINT *NameLengthPtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLGetData(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, + SQLPOINTER TargetValue, SQLLEN BufferLength, + SQLLEN *StrLen_or_IndPtr); + +SQLRETURN SQL_API SQLGetData(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, + SQLPOINTER TargetValue, SQLLEN BufferLength, + SQLLEN *StrLen_or_IndPtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLGetData(StatementHandle, ColumnNumber, TargetType, TargetValue, BufferLength, StrLen_or_IndPtr)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLGetDescField(SQLHDESC DescriptorHandle, + SQLSMALLINT RecNumber, SQLSMALLINT FieldIdentifier, + SQLPOINTER Value, SQLINTEGER BufferLength, + SQLINTEGER *StringLength) +{ + SQLRETURN r; + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLGetDescRec(SQLHDESC DescriptorHandle, + SQLSMALLINT RecNumber, SQLCHAR *Name, + SQLSMALLINT BufferLength, SQLSMALLINT *StringLengthPtr, + SQLSMALLINT *TypePtr, SQLSMALLINT *SubTypePtr, + SQLLEN *LengthPtr, SQLSMALLINT *PrecisionPtr, + SQLSMALLINT *ScalePtr, SQLSMALLINT *NullablePtr) +{ + SQLRETURN r; + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLGetDiagField(SQLSMALLINT HandleType, SQLHANDLE Handle, SQLSMALLINT RecNumber, SQLSMALLINT DiagIdentifier, + SQLPOINTER DiagInfo, SQLSMALLINT BufferLength, SQLSMALLINT *StringLength); + +SQLRETURN SQL_API SQLGetDiagField(SQLSMALLINT HandleType, SQLHANDLE Handle, SQLSMALLINT RecNumber, SQLSMALLINT DiagIdentifier, + SQLPOINTER DiagInfo, SQLSMALLINT BufferLength, SQLSMALLINT *StringLength) +{ + SQLRETURN r; + PROFILE(r, r = doSQLGetDiagField(HandleType, Handle, RecNumber, DiagIdentifier, + DiagInfo, BufferLength, StringLength)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLGetDiagRec(SQLSMALLINT HandleType, SQLHANDLE Handle, SQLSMALLINT RecNumber, + SQLCHAR *Sqlstate, SQLINTEGER *NativeError, SQLCHAR* MessageText, SQLSMALLINT BufferLength, SQLSMALLINT *TextLength); + +SQLRETURN SQL_API SQLGetDiagRec(SQLSMALLINT HandleType, SQLHANDLE Handle, SQLSMALLINT RecNumber, + SQLCHAR *Sqlstate, SQLINTEGER *NativeError, SQLCHAR* MessageText, SQLSMALLINT BufferLength, SQLSMALLINT *TextLength) +{ + SQLRETURN r; + PROFILE(r, r = doSQLGetDiagRec(HandleType, Handle, RecNumber, Sqlstate, NativeError, MessageText, BufferLength, TextLength)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLGetEnvAttr(SQLHENV EnvironmentHandle, SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength); + +SQLRETURN SQL_API SQLGetEnvAttr(SQLHENV EnvironmentHandle, SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_ENV, EnvironmentHandle); + PROFILE(r, r = doSQLGetEnvAttr(EnvironmentHandle, Attribute, Value, BufferLength, StringLength)); + todbc_tls_buf_reclaim(); + return r; +} + +// SQLRETURN SQL_API SQLGetFunctions(SQLHDBC ConnectionHandle, SQLUSMALLINT FunctionId, SQLUSMALLINT *Supported) +// { +// SQLRETURN r; +// errs_clear(SQL_HANDLE_DBC, ConnectionHandle); +// PROFILE(r, r = SQL_ERROR); +// todbc_tls_buf_reclaim(); +// return r; +// } + +static SQLRETURN doSQLGetInfo(SQLHDBC ConnectionHandle, SQLUSMALLINT InfoType, SQLPOINTER InfoValue, + SQLSMALLINT BufferLength, SQLSMALLINT *StringLengthPtr); + +SQLRETURN SQL_API SQLGetInfo(SQLHDBC ConnectionHandle, SQLUSMALLINT InfoType, SQLPOINTER InfoValue, + SQLSMALLINT BufferLength, SQLSMALLINT *StringLengthPtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = doSQLGetInfo(ConnectionHandle, InfoType, InfoValue, BufferLength, StringLengthPtr)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLGetStmtAttr(SQLHSTMT StatementHandle, SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength); + +SQLRETURN SQL_API SQLGetStmtAttr(SQLHSTMT StatementHandle, SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLGetStmtAttr(StatementHandle, Attribute, Value, BufferLength, StringLength)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLGetStmtOption(SQLHSTMT StatementHandle, + SQLUSMALLINT Option, SQLPOINTER Value) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLGetTypeInfo(SQLHSTMT StatementHandle, SQLSMALLINT DataType); + +SQLRETURN SQL_API SQLGetTypeInfo(SQLHSTMT StatementHandle, SQLSMALLINT DataType) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLGetTypeInfo(StatementHandle, DataType)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLNumResultCols(SQLHSTMT StatementHandle, SQLSMALLINT *ColumnCount); + +SQLRETURN SQL_API SQLNumResultCols(SQLHSTMT StatementHandle, SQLSMALLINT *ColumnCount) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLNumResultCols(StatementHandle, ColumnCount)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLParamData(SQLHSTMT StatementHandle, SQLPOINTER *Value) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLPrepare(SQLHSTMT StatementHandle, SQLCHAR* StatementText, SQLINTEGER TextLength); + +SQLRETURN SQL_API SQLPrepare(SQLHSTMT StatementHandle, SQLCHAR* StatementText, SQLINTEGER TextLength) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLPrepare(StatementHandle, StatementText, TextLength)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLPutData(SQLHSTMT StatementHandle, SQLPOINTER Data, SQLLEN StrLen_or_Ind) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLRowCount(SQLHSTMT StatementHandle, SQLLEN* RowCount); + +SQLRETURN SQL_API SQLRowCount(SQLHSTMT StatementHandle, SQLLEN* RowCount) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLRowCount(StatementHandle, RowCount)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLSetConnectAttr(SQLHDBC ConnectionHandle, SQLINTEGER Attribute, SQLPOINTER Value, SQLINTEGER StringLength) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLSetConnectOption(SQLHDBC ConnectionHandle, SQLUSMALLINT Option, SQLULEN Value) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLSetCursorName(SQLHSTMT StatementHandle, SQLCHAR* CursorName, SQLSMALLINT NameLength) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLSetDescField(SQLHDESC DescriptorHandle, SQLSMALLINT RecNumber, SQLSMALLINT FieldIdentifier, + SQLPOINTER Value, SQLINTEGER BufferLength) +{ + SQLRETURN r; + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLSetDescRec(SQLHDESC DescriptorHandle, SQLSMALLINT RecNumber, SQLSMALLINT Type, + SQLSMALLINT SubType, SQLLEN Length, SQLSMALLINT Precision, SQLSMALLINT Scale, + SQLPOINTER Data, SQLLEN *StringLength, SQLLEN *Indicator) +{ + SQLRETURN r; + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLSetEnvAttr(SQLHENV EnvironmentHandle, SQLINTEGER Attribute, SQLPOINTER Value, SQLINTEGER StringLength); + +SQLRETURN SQL_API SQLSetEnvAttr(SQLHENV EnvironmentHandle, SQLINTEGER Attribute, SQLPOINTER Value, SQLINTEGER StringLength) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_ENV, EnvironmentHandle); + PROFILE(r, r = doSQLSetEnvAttr(EnvironmentHandle, Attribute, Value, StringLength)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLSetParam(SQLHSTMT StatementHandle, SQLUSMALLINT ParameterNumber, SQLSMALLINT ValueType, + SQLSMALLINT ParameterType, SQLULEN LengthPrecision, SQLSMALLINT ParameterScale, SQLPOINTER ParameterValue, + SQLLEN *StrLen_or_IndPtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLSetStmtAttr(SQLHSTMT StatementHandle, SQLINTEGER Attribute, SQLPOINTER Value, SQLINTEGER StringLength); + +SQLRETURN SQL_API SQLSetStmtAttr(SQLHSTMT StatementHandle, SQLINTEGER Attribute, SQLPOINTER Value, SQLINTEGER StringLength) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLSetStmtAttr(StatementHandle, Attribute, Value, StringLength)); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLSetStmtOption(SQLHSTMT StatementHandle, SQLUSMALLINT Option, SQLULEN Value) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLSpecialColumns(SQLHSTMT StatementHandle, SQLUSMALLINT IdentifierType, + SQLCHAR *CatalogName, SQLSMALLINT NameLength1, + SQLCHAR *SchemaName, SQLSMALLINT NameLength2, + SQLCHAR *TableName, SQLSMALLINT NameLength3, + SQLUSMALLINT Scope, SQLUSMALLINT Nullable) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLStatistics(SQLHSTMT StatementHandle, + SQLCHAR *CatalogName, SQLSMALLINT NameLength1, + SQLCHAR *SchemaName, SQLSMALLINT NameLength2, + SQLCHAR *TableName, SQLSMALLINT NameLength3, + SQLUSMALLINT Unique, SQLUSMALLINT Reserved) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLTables(SQLHSTMT StatementHandle, + SQLCHAR *CatalogName, SQLSMALLINT NameLength1, + SQLCHAR *SchemaName, SQLSMALLINT NameLength2, + SQLCHAR *TableName, SQLSMALLINT NameLength3, + SQLCHAR *TableType, SQLSMALLINT NameLength4) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + +SQLRETURN SQL_API SQLTransact(SQLHENV EnvironmentHandle, SQLHDBC ConnectionHandle, SQLUSMALLINT CompletionType) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_ENV, EnvironmentHandle); + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = SQL_ERROR); + todbc_tls_buf_reclaim(); + return r; +} + + + + +static SQLRETURN doSQLDriverConnect( + SQLHDBC ConnectionHandle, + SQLHWND WindowHandle, + SQLCHAR *InConnectionString, + SQLSMALLINT StringLength1, + SQLCHAR *OutConnectionString, + SQLSMALLINT BufferLength, + SQLSMALLINT *StringLength2Ptr, + SQLUSMALLINT DriverCompletion); + +SQLRETURN SQL_API SQLDriverConnect( + SQLHDBC ConnectionHandle, + SQLHWND WindowHandle, + SQLCHAR *InConnectionString, + SQLSMALLINT StringLength1, + SQLCHAR *OutConnectionString, + SQLSMALLINT BufferLength, + SQLSMALLINT *StringLength2Ptr, + SQLUSMALLINT DriverCompletion) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_DBC, ConnectionHandle); + PROFILE(r, r = doSQLDriverConnect(ConnectionHandle, WindowHandle, InConnectionString, StringLength1, + OutConnectionString, BufferLength, StringLength2Ptr, DriverCompletion)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLBindParameter( + SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, + SQLSMALLINT InputOutputType, + SQLSMALLINT ValueType, + SQLSMALLINT ParameterType, + SQLULEN ColumnSize, + SQLSMALLINT DecimalDigits, + SQLPOINTER ParameterValuePtr, + SQLLEN BufferLength, + SQLLEN * StrLen_or_IndPtr); + +SQLRETURN SQL_API SQLBindParameter( + SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, + SQLSMALLINT InputOutputType, + SQLSMALLINT ValueType, + SQLSMALLINT ParameterType, + SQLULEN ColumnSize, + SQLSMALLINT DecimalDigits, + SQLPOINTER ParameterValuePtr, + SQLLEN BufferLength, + SQLLEN * StrLen_or_IndPtr) +{ + SQLRETURN r; + P("ParameterNumber:[%d]; InputOutputType:[%d]%s; ValueType:[%d]%s; ParameterType:[%d]%s; " + "ColumnSize:[%ld]; DecimalDigits:[%d]; ParameterValuePtr:[%p]; BufferLength:[%ld]; StrLen_or_IndPtr:[%p]", + ParameterNumber, InputOutputType, sql_input_output_type(InputOutputType), + ValueType, sql_c_type(ValueType), + ParameterType, sql_sql_type(ParameterType), + ColumnSize, DecimalDigits, ParameterValuePtr, BufferLength, StrLen_or_IndPtr); + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLBindParameter(StatementHandle, ParameterNumber, InputOutputType, ValueType, ParameterType, + ColumnSize, DecimalDigits, ParameterValuePtr, BufferLength, StrLen_or_IndPtr)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLNumParams(SQLHSTMT StatementHandle, SQLSMALLINT *ParameterCountPtr); + +SQLRETURN SQL_API SQLNumParams(SQLHSTMT StatementHandle, SQLSMALLINT *ParameterCountPtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLNumParams(StatementHandle, ParameterCountPtr)); + todbc_tls_buf_reclaim(); + return r; +} + +static SQLRETURN doSQLDescribeParam( + SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, + SQLSMALLINT * DataTypePtr, + SQLULEN * ParameterSizePtr, + SQLSMALLINT * DecimalDigitsPtr, + SQLSMALLINT * NullablePtr); + +SQLRETURN SQL_API SQLDescribeParam( + SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, + SQLSMALLINT * DataTypePtr, + SQLULEN * ParameterSizePtr, + SQLSMALLINT * DecimalDigitsPtr, + SQLSMALLINT * NullablePtr) +{ + SQLRETURN r; + errs_clear(SQL_HANDLE_STMT, StatementHandle); + PROFILE(r, r = doSQLDescribeParam(StatementHandle, ParameterNumber, DataTypePtr, + ParameterSizePtr, DecimalDigitsPtr, NullablePtr)); + todbc_tls_buf_reclaim(); + return r; +} + + + + + +SQLRETURN SQL_API SQLBrowseConnect( + SQLHDBC hdbc, + SQLCHAR *szConnStrIn, + SQLSMALLINT cchConnStrIn, + SQLCHAR *szConnStrOut, + SQLSMALLINT cchConnStrOutMax, + SQLSMALLINT *pcchConnStrOut); + +SQLRETURN SQL_API SQLBulkOperations( + SQLHSTMT StatementHandle, + SQLSMALLINT Operation); + +SQLRETURN SQL_API SQLColAttributes( + SQLHSTMT hstmt, + SQLUSMALLINT icol, + SQLUSMALLINT fDescType, + SQLPOINTER rgbDesc, + SQLSMALLINT cbDescMax, + SQLSMALLINT *pcbDesc, + SQLLEN * pfDesc); + +SQLRETURN SQL_API SQLColumnPrivileges( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cchCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cchSchemaName, + SQLCHAR *szTableName, + SQLSMALLINT cchTableName, + SQLCHAR *szColumnName, + SQLSMALLINT cchColumnName); + +SQLRETURN SQL_API SQLExtendedFetch( + SQLHSTMT hstmt, + SQLUSMALLINT fFetchType, + SQLLEN irow, + SQLULEN *pcrow, + SQLUSMALLINT *rgfRowStatus); + +SQLRETURN SQL_API SQLForeignKeys( + SQLHSTMT hstmt, + SQLCHAR *szPkCatalogName, + SQLSMALLINT cchPkCatalogName, + SQLCHAR *szPkSchemaName, + SQLSMALLINT cchPkSchemaName, + SQLCHAR *szPkTableName, + SQLSMALLINT cchPkTableName, + SQLCHAR *szFkCatalogName, + SQLSMALLINT cchFkCatalogName, + SQLCHAR *szFkSchemaName, + SQLSMALLINT cchFkSchemaName, + SQLCHAR *szFkTableName, + SQLSMALLINT cchFkTableName); + +SQLRETURN SQL_API SQLMoreResults( + SQLHSTMT hstmt); + +SQLRETURN SQL_API SQLNativeSql +( + SQLHDBC hdbc, + SQLCHAR* szSqlStrIn, + SQLINTEGER cchSqlStrIn, + SQLCHAR* szSqlStr, + SQLINTEGER cchSqlStrMax, + SQLINTEGER *pcbSqlStr +); + +SQLRETURN SQL_API SQLParamOptions( + SQLHSTMT hstmt, + SQLULEN crow, + SQLULEN *pirow); + +SQLRETURN SQL_API SQLPrimaryKeys( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cchCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cchSchemaName, + SQLCHAR *szTableName, + SQLSMALLINT cchTableName); + +SQLRETURN SQL_API SQLProcedureColumns( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cchCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cchSchemaName, + SQLCHAR *szProcName, + SQLSMALLINT cchProcName, + SQLCHAR *szColumnName, + SQLSMALLINT cchColumnName); + +SQLRETURN SQL_API SQLProcedures( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cchCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cchSchemaName, + SQLCHAR *szProcName, + SQLSMALLINT cchProcName); + + + +SQLRETURN SQL_API SQLSetPos( + SQLHSTMT hstmt, + SQLSETPOSIROW irow, + SQLUSMALLINT fOption, + SQLUSMALLINT fLock); + +SQLRETURN SQL_API SQLTablePrivileges( + SQLHSTMT hstmt, + SQLCHAR *szCatalogName, + SQLSMALLINT cchCatalogName, + SQLCHAR *szSchemaName, + SQLSMALLINT cchSchemaName, + SQLCHAR *szTableName, + SQLSMALLINT cchTableName); + +SQLRETURN SQL_API SQLDrivers( + SQLHENV henv, + SQLUSMALLINT fDirection, + SQLCHAR *szDriverDesc, + SQLSMALLINT cchDriverDescMax, + SQLSMALLINT *pcchDriverDesc, + SQLCHAR *szDriverAttributes, + SQLSMALLINT cchDrvrAttrMax, + SQLSMALLINT *pcchDrvrAttr); + +SQLRETURN SQL_API SQLAllocHandleStd( + SQLSMALLINT fHandleType, + SQLHANDLE hInput, + SQLHANDLE *phOutput); +SQLRETURN SQL_API SQLSetScrollOptions( + SQLHSTMT hstmt, + SQLUSMALLINT fConcurrency, + SQLLEN crowKeyset, + SQLUSMALLINT crowRowset); + + + + +static SQLRETURN doSQLAllocEnv(SQLHENV *EnvironmentHandle) +{ + env_t *env = (env_t*)calloc(1, sizeof(*env)); + OILE(env, ""); + + int r = env_init(env); + if (r) return SQL_ERROR; + + *EnvironmentHandle = env; + + return SQL_SUCCESS; +} + +static SQLRETURN doSQLAllocHandle(SQLSMALLINT HandleType, SQLHANDLE InputHandle, SQLHANDLE *OutputHandle) +{ + P("HandleType:[%d]%s", HandleType, sql_handle_type(HandleType)); + switch (HandleType) + { + case SQL_HANDLE_ENV: + { + return doSQLAllocEnv(OutputHandle); + } break; + case SQL_HANDLE_DBC: + { + errs_clear(SQL_HANDLE_ENV, InputHandle); + return doSQLAllocConnect(InputHandle, OutputHandle); + } break; + case SQL_HANDLE_STMT: + { + errs_clear(SQL_HANDLE_DBC, InputHandle); + return doSQLAllocStmt(InputHandle, OutputHandle); + } break; + default: + { + ONIY(0, ""); + } break; + } +} + +static SQLRETURN doSQLSetEnvAttr(SQLHENV EnvironmentHandle, SQLINTEGER Attribute, SQLPOINTER Value, SQLINTEGER StringLength) +{ + P("Attribute:[%d]%s", Attribute, sql_env_attr_type(Attribute)); + env_t *env = (env_t*)EnvironmentHandle; + OILE(env, ""); + + switch (Attribute) + { + case SQL_ATTR_ODBC_VERSION: + { + int32_t ver = (int32_t)(size_t)Value; + P("client odbc ver:[%d]", ver); + if (ver < env->odbc_ver) { + // fall back to lower version + env->odbc_ver = ver; + } + return SQL_SUCCESS; + } break; + default: + { + ONIY(0, ""); + } break; + } +} + +static SQLRETURN doSQLGetEnvAttr(SQLHENV EnvironmentHandle, SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength) +{ + P("Attribute:[%d]%s; Value:[%p]; BufferLength:[%d]; StringLength:[%p][%d]", + Attribute, sql_env_attr_type(Attribute), Value, BufferLength, + StringLength, StringLength ? *StringLength : 0); + env_t *env = (env_t*)EnvironmentHandle; + OILE(env, ""); + + switch (Attribute) + { + case SQL_ATTR_ODBC_VERSION: + { + *(int32_t*)Value = env->odbc_ver; + P("odbc ver:[%d]", env->odbc_ver); + return SQL_SUCCESS; + } break; + default: + { + ONIY(0, ""); + } break; + } +} + +static SQLRETURN doSQLAllocConnect(SQLHENV EnvironmentHandle, SQLHDBC *ConnectionHandle) +{ + env_t *env = (env_t*)EnvironmentHandle; + OILE(env, ""); + errs_t *errs = env_get_errs(env); + + conn_t *conn = conn_new(env); + if (!conn) { + SET_OOM(errs, "alloc conn failed"); + return SQL_ERROR; + } + + do { + *ConnectionHandle = conn; + + return SQL_SUCCESS; + } while (0); + + conn_free(conn); + + return SQL_ERROR; +} + +static SQLRETURN doSQLFreeHandle(SQLSMALLINT HandleType, SQLHANDLE Handle) +{ + switch (HandleType) + { + case SQL_HANDLE_ENV: + { + return doSQLFreeEnv(Handle); + } break; + case SQL_HANDLE_DBC: + { + return doSQLFreeConnect(Handle); + } break; + case SQL_HANDLE_STMT: + { + stmt_t *stmt = (stmt_t*)Handle; + stmt_free(stmt); + return SQL_SUCCESS; + } break; + default: + { + ONIY(0, "HandleType:[%d]%s", HandleType, sql_handle_type(HandleType)); + } break; + } +} + +static SQLRETURN do_connect(conn_t *conn) { + errs_t *errs = &conn->errs; + + OD("enc:src/char/wchar/db/locale:[%s/%s/%s/%s/%s]", + conn->enc_src, conn->enc_char, conn->enc_wchar, conn->enc_db, conn->enc_locale); + + SQLRETURN r; + + r = conn_check_charset(conn, errs); + if (r!=SQL_SUCCESS) return r; + + if (0) { + // test with null_conn + if (conn_init_null_conn(conn)) { + SET_GENERAL(errs, "failed to init null conn for test"); + return SQL_ERROR; + } + } else { + if (conn_init_tsdb_conn(conn)) { + SET_GENERAL(errs, "failed to init taos conn for test"); + return SQL_ERROR; + } + } + + return conn_connect(conn); +} + +static SQLRETURN doSQLConnect(SQLHDBC ConnectionHandle, + SQLCHAR *ServerName, SQLSMALLINT NameLength1, + SQLCHAR *UserName, SQLSMALLINT NameLength2, + SQLCHAR *Authentication, SQLSMALLINT NameLength3) +{ + conn_t *conn = (conn_t*)ConnectionHandle; + OILE(conn, ""); + ONIY(ServerName, ""); + + errs_t *errs = &conn->errs; + + const char *enc_to = conn->enc_locale; + const char *enc_from = conn->enc_char; + + SQLRETURN ok = SQL_ERROR; + + conn_val_t *val = &conn->val; + conn_val_reset(val); + + do { + if (ServerName) { + size_t slen = (size_t)NameLength1; + todbc_string_t dsn = todbc_tls_conv(NULL, enc_to, enc_from, (const unsigned char*)ServerName, &slen); + if (!dsn.buf) { + SET_OOM(errs, "alloc buf failed"); + return SQL_ERROR; + } + val->dsn = strdup((const char*)dsn.buf); + if (!val->dsn) { + SET_OOM(errs, "strdup failed"); + return SQL_ERROR; + } + } + if (UserName) { + size_t slen = (size_t)NameLength2; + todbc_string_t uid = todbc_tls_conv(NULL, enc_to, enc_from, (const unsigned char*)UserName, &slen); + if (!uid.buf) { + SET_OOM(errs, "alloc buf failed"); + return SQL_ERROR; + } + val->uid = strdup((const char*)uid.buf); + if (!val->uid) { + SET_OOM(errs, "strdup failed"); + return SQL_ERROR; + } + } + if (Authentication) { + size_t slen = (size_t)NameLength3; + todbc_string_t pwd = todbc_tls_conv(NULL, enc_to, enc_from, (const unsigned char*)Authentication, &slen); + if (!pwd.buf) { + SET_OOM(errs, "alloc buf failed"); + return SQL_ERROR; + } + val->pwd = strdup((const char*)pwd.buf); + if (!val->pwd) { + SET_OOM(errs, "strdup failed"); + return SQL_ERROR; + } + } + + OD("................."); + ok = do_connect(conn); + OD("................."); + if (ok!=SQL_SUCCESS) break; + + CONN_SET_CONNECTED(conn); + return SQL_SUCCESS; + } while (0); + + conn_val_reset(val); + + return ok; +} + +static SQLRETURN doSQLDriverConnect( + SQLHDBC ConnectionHandle, + SQLHWND WindowHandle, + SQLCHAR *InConnectionString, + SQLSMALLINT StringLength1, + SQLCHAR *OutConnectionString, + SQLSMALLINT BufferLength, + SQLSMALLINT *StringLength2Ptr, + SQLUSMALLINT DriverCompletion) +{ + conn_t *conn = (conn_t*)ConnectionHandle; + OILE(conn, ""); + ONIY(InConnectionString, ""); + + errs_t *errs = &conn->errs; + +#ifndef _MSC_VER + if (DriverCompletion!=SQL_DRIVER_NOPROMPT) { + SET_NIY(errs, "option[%d] other than SQL_DRIVER_NOPROMPT not supported yet", DriverCompletion); + return SQL_ERROR; + } +#endif + + const char *enc_to = conn->enc_locale; + const char *enc_from = conn->enc_char; + + size_t slen = (size_t)StringLength1; + todbc_string_t ts = todbc_tls_conv(NULL, enc_to, enc_from, (const unsigned char*)InConnectionString, &slen); + const char *connStr = (const char*)ts.buf; + if (!connStr) { + SET_OOM(errs, "alloc buf failed"); + return SQL_ERROR; + } + + SQLRETURN ok = SQL_ERROR; + + conn_val_t *val = &conn->val; + conn_val_reset(val); + + do { + // TO_DO: genralize + int n = todbc_parse_conn_string(connStr, val); + if (n) { + SET_GENERAL(errs, "unrecognized connection string:[%s]", connStr); + break; + } + + todbc_enc_t enc; + if (val->enc_char) { + enc = todbc_tls_iconv_enc(val->enc_char); + if (enc.enc[0]=='\0') { + SET_GENERAL(errs, "unrecognized charset:[%s]", val->enc_char); + break; + } + snprintf(conn->enc_char, sizeof(conn->enc_char), "%s", val->enc_char); + } + if (val->enc_wchar) { + enc = todbc_tls_iconv_enc(val->enc_wchar); + if (enc.enc[0]=='\0') { + SET_GENERAL(errs, "unrecognized charset:[%s]", val->enc_wchar); + break; + } + snprintf(conn->enc_wchar, sizeof(conn->enc_wchar), "%s", val->enc_wchar); + } + if (val->enc_db) { + enc = todbc_tls_iconv_enc(val->enc_db); + if (enc.enc[0]=='\0') { + SET_GENERAL(errs, "unrecognized charset:[%s]", val->enc_db); + break; + } + snprintf(conn->enc_db, sizeof(conn->enc_db), "%s", val->enc_db); + } + if (val->enc_local) { + enc = todbc_tls_iconv_enc(val->enc_local); + if (enc.enc[0]=='\0') { + SET_GENERAL(errs, "unrecognized charset:[%s]", val->enc_local); + break; + } + snprintf(conn->enc_locale, sizeof(conn->enc_locale), "%s", val->enc_local); + } + + ok = do_connect(conn); + if (ok!=SQL_SUCCESS) break; + ok = SQL_ERROR; + + n = 0; + if (OutConnectionString) { + n = snprintf((char*)OutConnectionString, (size_t)BufferLength, "%s", connStr); + } + if (StringLength2Ptr) { + *StringLength2Ptr = (SQLSMALLINT)n; + } + + CONN_SET_CONNECTED(conn); + return SQL_SUCCESS; + } while (0); + + conn_val_reset(val); + + return ok; +} + +static SQLRETURN doSQLGetInfo(SQLHDBC ConnectionHandle, SQLUSMALLINT InfoType, SQLPOINTER InfoValue, + SQLSMALLINT BufferLength, SQLSMALLINT *StringLengthPtr) +{ + P("InfoType:[%d]%s; BufferLength:[%d]; StringLengthPtr:[%p]%d", + InfoType, sql_info_type(InfoType), BufferLength, + StringLengthPtr, StringLengthPtr ? *StringLengthPtr : 0); + + switch (InfoType) + { + case SQL_DRIVER_ODBC_VER: + { + // how to sync with env->odbc_ver ? + const char *v = "03.00"; + int n = snprintf((char*)InfoValue, (size_t)BufferLength, "%s", v); + OILE(n>0, ""); + *StringLengthPtr = (SQLSMALLINT)n; + } break; + case SQL_DESCRIBE_PARAMETER: + { + const char *v = "Y"; + int n = snprintf((char*)InfoValue, (size_t)BufferLength, "%s", v); + OILE(n>0, ""); + *StringLengthPtr = (SQLSMALLINT)n; + } break; + case SQL_NEED_LONG_DATA_LEN: + { + const char *v = "Y"; + int n = snprintf((char*)InfoValue, (size_t)BufferLength, "%s", v); + OILE(n>0, ""); + *StringLengthPtr = (SQLSMALLINT)n; + } break; + case SQL_MAX_COLUMN_NAME_LEN: + { + SQLUSMALLINT v = 64; + OILE(BufferLength==sizeof(v), ""); + *(SQLUSMALLINT*)InfoValue = v; + if (StringLengthPtr) *StringLengthPtr = sizeof(v); + } break; + case SQL_TXN_ISOLATION_OPTION: + { + SQLUINTEGER v = SQL_TXN_READ_UNCOMMITTED; + OILE(BufferLength==sizeof(v), ""); + *(SQLUINTEGER*)InfoValue = v; + if (StringLengthPtr) *StringLengthPtr = sizeof(v); + } break; + case SQL_CURSOR_COMMIT_BEHAVIOR: + { + SQLUSMALLINT v = SQL_CB_PRESERVE; + OILE(BufferLength==sizeof(v), ""); + *(SQLUSMALLINT*)InfoValue = v; + if (StringLengthPtr) *StringLengthPtr = sizeof(v); + } break; + case SQL_CURSOR_ROLLBACK_BEHAVIOR: + { + SQLUSMALLINT v = SQL_CB_PRESERVE; + OILE(BufferLength==sizeof(v), ""); + *(SQLUSMALLINT*)InfoValue = v; + if (StringLengthPtr) *StringLengthPtr = sizeof(v); + } break; + case SQL_GETDATA_EXTENSIONS: + { + SQLUINTEGER v = SQL_GD_ANY_COLUMN; + OILE(BufferLength==sizeof(v), ""); + *(SQLUINTEGER*)InfoValue = v; + if (StringLengthPtr) *StringLengthPtr = sizeof(v); + } break; + case SQL_DTC_TRANSITION_COST: + { + SQLUINTEGER v = SQL_DTC_ENLIST_EXPENSIVE | SQL_DTC_UNENLIST_EXPENSIVE; + OILE(BufferLength==sizeof(v), ""); + *(SQLUINTEGER*)InfoValue = v; + if (StringLengthPtr) *StringLengthPtr = sizeof(v); + } break; + case SQL_MAX_CONCURRENT_ACTIVITIES: + { + SQLUSMALLINT v = 10240; + OILE(BufferLength==sizeof(v), ""); + *(SQLUSMALLINT*)InfoValue = v; + if (StringLengthPtr) *StringLengthPtr = sizeof(v); + } break; + default: + { + ONIY(0, ""); + // return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_fill_error(errs_t *errs, const char *enc_to, const char *enc_from, + SQLSMALLINT RecNumber, SQLCHAR *Sqlstate, SQLINTEGER *NativeError, + SQLCHAR* MessageText, SQLSMALLINT BufferLength, SQLSMALLINT *TextLength) +{ + if (errs_count(errs)<=0) return SQL_NO_DATA; + + const char *sql_state = NULL; + const char *err_str = NULL; + int r = errs_fetch(errs, RecNumber-1, &sql_state, &err_str); + if (r) return SQL_NO_DATA; + OILE(sql_state && err_str, ""); + const unsigned char *src = (const unsigned char*)err_str; + size_t slen = strlen(err_str); + unsigned char *dst = (unsigned char*)MessageText; + size_t dlen = (size_t)BufferLength; + // OILE(dst, ""); + if (!MessageText) { + OILE(TextLength, ""); + *TextLength = 4096; + return SQL_SUCCESS; + } + todbc_string_t s = todbc_tls_write(enc_to, enc_from, src, &slen, dst, dlen); + *NativeError = 0; + *TextLength = (SQLSMALLINT)s.bytes; + snprintf((char*)Sqlstate, 6, "%s", sql_state); + return SQL_SUCCESS; +} + +static SQLRETURN doSQLGetDiagRec(SQLSMALLINT HandleType, SQLHANDLE Handle, SQLSMALLINT RecNumber, + SQLCHAR *Sqlstate, SQLINTEGER *NativeError, SQLCHAR* MessageText, SQLSMALLINT BufferLength, SQLSMALLINT *TextLength) +{ + OILE(RecNumber>0, ""); + OILE(Sqlstate, ""); + // OILE(NativeError, ""); + OILE(TextLength, ""); + switch (HandleType) + { + // case SQL_HANDLE_ENV: + // { + // env_t *env = (env_t*)Handle; + // FILL_ERROR(env); + // return SQL_SUCCESS; + // } break; + case SQL_HANDLE_DBC: + { + conn_t *conn = (conn_t*)Handle; + OILE(conn, ""); + errs_t *errs = conn_get_errs(conn); + OILE(errs, ""); + const char *enc_to = conn->enc_char; + const char *enc_from = conn->enc_src; + return do_fill_error(errs, enc_to, enc_from, RecNumber, Sqlstate, NativeError, MessageText, BufferLength, TextLength); + } break; + case SQL_HANDLE_STMT: + { + stmt_t *stmt = (stmt_t*)Handle; + OILE(stmt, ""); + if (!stmt->owner) return SQL_NO_DATA; + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + errs_t *errs = stmt_get_errs(stmt); + OILE(errs, ""); + const char *enc_to = conn->enc_char; + const char *enc_from = conn->enc_src; + return do_fill_error(errs, enc_to, enc_from, RecNumber, Sqlstate, NativeError, MessageText, BufferLength, TextLength); + } break; + default: + { + ONIY(0, "HandleType:[%d]%s", HandleType, sql_handle_type(HandleType)); + // return SQL_ERROR; + } break; + } +} + +static SQLRETURN doSQLAllocStmt(SQLHDBC ConnectionHandle, SQLHSTMT *StatementHandle) +{ + conn_t *conn = (conn_t*)ConnectionHandle; + OILE(conn, ""); + + errs_t *errs = &conn->errs; + + stmt_t *stmt = stmt_new(conn); + if (!stmt) { + SET_OOM(errs, "alloc stmt failed"); + return SQL_ERROR; + } + + do { + if (!StatementHandle) break; + + *StatementHandle = stmt; + return SQL_SUCCESS; + } while (0); + + stmt_free(stmt); + OILE(0, ""); +} + +static SQLRETURN doSQLFreeStmt(SQLHSTMT StatementHandle, SQLUSMALLINT Option) +{ + P("Option:[%d]%s", Option, sql_freestmt_option_type(Option)); + + switch (Option) + { + case SQL_CLOSE: + { + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + if (STMT_IS_NORM(stmt)) break; + return doSQLCloseCursor(StatementHandle); + } break; + // case SQL_UNBIND: + // { + // } break; + case SQL_RESET_PARAMS: + { + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + stmt_reset_params(stmt); + } break; + default: + { + ONIY(0, ""); + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN doSQLGetStmtAttr(SQLHSTMT StatementHandle, SQLINTEGER Attribute, SQLPOINTER Value, + SQLINTEGER BufferLength, SQLINTEGER *StringLength) +{ + P("Attribute:[%d]%s; BufferLength:[%d]; StringLength:[%p][%d]", + Attribute, sql_stmt_attr_type(Attribute), BufferLength, + StringLength, StringLength ? *StringLength : 0); + stmt_t *stmt = (stmt_t*)StatementHandle; + descs_t *descs = &stmt->descs; + + switch (Attribute) + { + case SQL_ATTR_APP_ROW_DESC: + { + OILE(BufferLength==sizeof(descs->app_row) || BufferLength==SQL_IS_POINTER, ""); + *(void**)Value = descs->app_row; + } break; + case SQL_ATTR_APP_PARAM_DESC: + { + OILE(BufferLength==sizeof(descs->app_param) || BufferLength==SQL_IS_POINTER, ""); + *(void**)Value = descs->app_param; + } break; + case SQL_ATTR_IMP_ROW_DESC: + { + OILE(BufferLength==sizeof(descs->imp_row) || BufferLength==SQL_IS_POINTER, ""); + *(void**)Value = descs->imp_row; + } break; + case SQL_ATTR_IMP_PARAM_DESC: + { + OILE(BufferLength==sizeof(descs->imp_param) || BufferLength==SQL_IS_POINTER, ""); + *(void**)Value = descs->imp_param; + } break; + default: + { + ONIY(0, ""); + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN doSQLExecDirect(SQLHSTMT StatementHandle, SQLCHAR* StatementText, SQLINTEGER TextLength) +{ + P("TextLength:[%d]%s", TextLength, sql_soi_type(TextLength)); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(!STMT_IS_EXECUTING(stmt), ""); + stmt_close_rs(stmt); + OILE(STMT_IS_NORM(stmt), ""); + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + errs_t *errs = &stmt->errs; + + const char *enc = conn->enc_char; + + OILE(StatementText, ""); + OILE(TextLength>=0 || TextLength==SQL_NTS, ""); + size_t slen = (TextLength>=0) ? (size_t)TextLength : (size_t)-1; + todbc_string_t sql = todbc_string_init(enc, StatementText, slen); + if (!sql.buf) { + SET_OOM(errs, ""); + return SQL_ERROR; + } + + return stmt_exec_direct(stmt, &sql); + + + // stmt_t *stmt = (stmt_t*)StatementHandle; + // rs_t *rs = &stmt->rs; + // rs_close(rs); + // OILE(rs->stmt==NULL); + + // if (1) { + // // taos_stmt_prepare fails to prepare a non-param-stmt-statement + // // thus we fall-back to use taos_query + // OILE(stmt->owner); + // conn_t *conn = stmt->owner->conn; + // OILE(conn); + + // OILE(rs->stmt==NULL); + // paramset_t *paramset = &stmt->paramset; + // paramset_close(paramset); + + // SQLRETURN e = stmt_set_statement(stmt, StatementText, TextLength); + // if (e!=SQL_SUCCESS) return e; + + // taos_rs_t *taos_rs = &rs->taos_rs; + + // const char *buf = (const char*)stmt->statement_db.buf; + // taos_rs->rs = taos_query(conn->taos, buf); + + // if (!taos_rs->rs) { + // int err = terrno; + // SET_ERROR(stmt, "HY000", "failed to execute statement:[%d]%s", err, tstrerror(err)); + // // keep executing/executed state unchanged + // return SQL_ERROR; + // } + + // taos_rs->owner = rs; + // taos_rs->rs_is_opened_by_taos_query = 1; + + // SET_EXECUTED(stmt); + // return stmt_after_exec(stmt); + // } else { + // SQLRETURN r = doSQLPrepare(StatementHandle, StatementText, TextLength); + // if (r!=SQL_SUCCESS) return r; + + // return doSQLExecute(StatementHandle); + // } +} + +static SQLRETURN doSQLRowCount(SQLHSTMT StatementHandle, SQLLEN* RowCount) +{ + OILE(RowCount, ""); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(STMT_IS_EXECUTED(stmt), ""); + + OILE(stmt->ext.get_affected_rows, ""); + SQLRETURN r = stmt->ext.get_affected_rows(stmt, RowCount); + if (r!=SQL_SUCCESS) return r; + + P("RowCount:[%ld]", *RowCount); + + return SQL_SUCCESS; +} + +static SQLRETURN doSQLNumResultCols(SQLHSTMT StatementHandle, SQLSMALLINT *ColumnCount) +{ + OILE(ColumnCount, ""); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(STMT_IS_EXECUTED(stmt), ""); + + OILE(stmt->ext.get_fields_count, ""); + SQLRETURN r = stmt->ext.get_fields_count(stmt, ColumnCount); + if (r!=SQL_SUCCESS) return r; + + P("ColumnCount:[%d]", *ColumnCount); + + return SQL_SUCCESS; +} + +static SQLRETURN doSQLDescribeCol(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLCHAR *ColumnName, SQLSMALLINT BufferLength, + SQLSMALLINT *NameLength, + SQLSMALLINT *DataType, SQLULEN *ColumnSize, + SQLSMALLINT *DecimalDigits, SQLSMALLINT *Nullable) +{ + P("ColumnNumber:[%d]; BufferLength:[%d]", ColumnNumber, BufferLength); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(STMT_IS_EXECUTED(stmt), ""); + + field_arg_t field_arg = {0}; + field_arg.ColumnNumber = ColumnNumber; + field_arg.ColumnName = ColumnName; + field_arg.BufferLength = BufferLength; + field_arg.NameLength = NameLength; + field_arg.DataType = DataType; + field_arg.ColumnSize = ColumnSize; + field_arg.DecimalDigits = DecimalDigits; + field_arg.Nullable = Nullable; + + OILE(stmt->ext.get_field, ""); + return stmt->ext.get_field(stmt, &field_arg); + + // sql_t *sql = (sql_t*)StatementHandle; + // rs_t *rs = &sql->rs; + // OILE(rs->sql==sql); + // OILE(BufferLength>=0); + // OILE(NameLength); + // OILE(DataType); + // OILE(ColumnSize); + // OILE(DecimalDigits); + // OILE(Nullable); + + // OILE(ColumnNumber>0 && ColumnNumber<=rs->n_fields); + // field_t *field = rs->fields + (ColumnNumber-1); + + // int n = snprintf((char*)ColumnName, (size_t)BufferLength, "%s", field->name); + // *NameLength = (SQLSMALLINT)n; + // *DataType = (SQLSMALLINT)field->data_type; + // *ColumnSize = (SQLUSMALLINT)field->col_size; + // *DecimalDigits = (SQLSMALLINT)field->decimal_digits; + // *Nullable = (SQLSMALLINT)field->nullable; + + + // P("ColumnNumber:[%d]; DataType:[%d]%s; Name:[%s]; ColumnSize:[%ld]; DecimalDigits:[%d]; Nullable:[%d]%s", + // ColumnNumber, *DataType, sql_sql_type(*DataType), field->name, + // *ColumnSize, *DecimalDigits, *Nullable, sql_nullable_type(*Nullable)); + + // return SQL_SUCCESS; +} + +static SQLRETURN doSQLDisconnect(SQLHDBC ConnectionHandle) +{ + conn_t *conn = (conn_t*)ConnectionHandle; + OILE(conn, ""); + OILE(CONN_IS_CONNECTED(conn), ""); + + conn_disconnect(conn); + + CONN_SET_NORM(conn); + return SQL_SUCCESS; + + + // OILE(conn->taos); + + // // conn/stmts which ever to close first? + // // as for ODBC, it's suggested to close connection first, then release all dangling statements + // // but we are not sure if doing so will result in memory leakage for taos + // // thus we choose to close dangling statements before shutdown connection + // conn_release_sqls(conn); + + // taos_close(conn->taos); + // conn->taos = 0; + + // return SQL_SUCCESS; +} + +static SQLRETURN doSQLPrepare(SQLHSTMT StatementHandle, SQLCHAR* StatementText, SQLINTEGER TextLength) +{ + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(!STMT_IS_EXECUTING(stmt), ""); + stmt_close_rs(stmt); + OILE(STMT_IS_NORM(stmt), ""); + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + errs_t *errs = &stmt->errs; + + const char *enc = conn->enc_char; + + OILE(StatementText, ""); + OILE(TextLength>=0 || TextLength==SQL_NTS, ""); + size_t slen = (TextLength>=0) ? (size_t)TextLength : (size_t)-1; + todbc_string_t sql = todbc_string_init(enc, StatementText, slen); + if (!sql.buf) { + SET_OOM(errs, ""); + return SQL_ERROR; + } + + if (1) { + todbc_string_t l = todbc_string_conv_to(&sql, conn->enc_src, NULL); + P("prepare:[%s]", (const char*)l.buf); + } + + return stmt_prepare(stmt, &sql); + + // sql_t *sql = (sql_t*)StatementHandle; + // rs_t *rs = &sql->rs; + + // OILE(rs->taos_rs.rs_is_opened_by_taos_query==0); + + // OILE(sql->owner); + // conn_t *conn = sql->owner->conn; + // OILE(conn); + + // OILE(rs->sql==NULL); + // paramset_t *paramset = &sql->paramset; + // paramset_close(paramset); + + // SQLRETURN r = sql_set_statement(sql, StatementText, TextLength); + // if (r!=SQL_SUCCESS) return r; + // + // int n_params = 0; + // r = sql_prepare(sql, &n_params); + // if (r!=SQL_SUCCESS) return r; + + // paramset_init_params(paramset, n_params); + // if (!paramset->params_cache || !paramset->params || !paramset->taos_binds) { + // sql_close_taos_stmt(sql); + // SET_ERROR(sql, "HY001", ""); + // return SQL_ERROR; + // } + + // // for (int i=0; iparams + i; + + // // } + + // paramset->sql = sql; + // paramset->n_params = n_params; + + // return sql_after_prepare(sql); +} + +static SQLRETURN doSQLNumParams(SQLHSTMT StatementHandle, SQLSMALLINT *ParameterCountPtr) +{ + OILE(ParameterCountPtr, ""); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(stmt->prepared, ""); + + *ParameterCountPtr = (SQLSMALLINT)stmt->paramset.n_params; + + P("ParameterCount:[%d]", *ParameterCountPtr); + + return SQL_SUCCESS; +} + +static SQLRETURN doSQLBindParameter( + SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, + SQLSMALLINT InputOutputType, + SQLSMALLINT ValueType, + SQLSMALLINT ParameterType, + SQLULEN ColumnSize, + SQLSMALLINT DecimalDigits, + SQLPOINTER ParameterValuePtr, + SQLLEN BufferLength, + SQLLEN * StrLen_or_IndPtr) +{ + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(!STMT_IS_EXECUTING(stmt), ""); + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + param_binding_t arg = {0}; + arg.ParameterNumber = ParameterNumber; + arg.InputOutputType = InputOutputType; + arg.ValueType = ValueType; // sql c data type + arg.ParameterType = ParameterType; // sql data type + arg.ColumnSize = ColumnSize; + arg.DecimalDigits = DecimalDigits; + arg.ParameterValuePtr = ParameterValuePtr; + arg.BufferLength = BufferLength; + arg.StrLen_or_IndPtr = StrLen_or_IndPtr; + + return stmt_bind_param(stmt, &arg); + + // sql_t *sql = (sql_t*)StatementHandle; + // rs_t *rs = &sql->rs; + // OILE(rs->sql==NULL); + // OILE(IS_PREPARED(sql)); + // paramset_t *paramset = &sql->paramset; + // OILE(ParameterNumber>0); + + // OILE(StrLen_or_IndPtr); + + // paramset_realloc_bps(paramset, ParameterNumber); + // if (!paramset->bps_cache || !paramset->bps || paramset->n_bpsbps + (ParameterNumber-1); + // bp->ParameterNumber = ParameterNumber; + // bp->InputOutputType = InputOutputType; + // bp->ValueType = ValueType; + // bp->ParameterType = ParameterType; + // bp->ColumnSize = ColumnSize; + // bp->DecimalDigits = DecimalDigits; + // bp->ParameterValuePtr = ParameterValuePtr; + // bp->BufferLength = BufferLength; + // bp->StrLen_or_IndPtr = StrLen_or_IndPtr; + + // return SQL_SUCCESS; +} + +static SQLRETURN doSQLSetStmtAttr(SQLHSTMT StatementHandle, SQLINTEGER Attribute, SQLPOINTER Value, SQLINTEGER StringLength) +{ + P("Attribute:[%d]%s; Value:[%p]; StringLength:[%d]%s", + Attribute, sql_stmt_attr_type(Attribute), Value, + StringLength, sql_soi_type(StringLength)); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(!STMT_IS_EXECUTING(stmt), ""); + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + stmt_attr_t *attr = &stmt->attr; + + errs_t *errs = &stmt->errs; + + switch (Attribute) + { + case SQL_ATTR_PARAM_BIND_TYPE: + { + SQLULEN v = (SQLULEN)Value; + ONIY(v!=SQL_PARAM_BIND_BY_COLUMN, ""); + attr->bind_type = v; + P("%s:[%ld]", sql_stmt_attr_type(Attribute), v); + } break; + case SQL_ATTR_PARAMSET_SIZE: + { + SQLULEN v = (SQLULEN)Value; + ONIY(v!=0, ""); + attr->paramset_size = v; + P("%s:[%ld]", sql_stmt_attr_type(Attribute), v); + } break; + case SQL_ATTR_PARAM_BIND_OFFSET_PTR: + { + SQLULEN *v = (SQLULEN*)Value; + attr->bind_offset_ptr = v; + P("%s:[%p]", sql_stmt_attr_type(Attribute), v); + } break; + default: + { + P("Attribute:[%d]%s not implemented yet", Attribute, sql_stmt_attr_type(Attribute)); + SET_NIY(errs, "Attribute:[%d]%s", Attribute, sql_stmt_attr_type(Attribute)); + // ONSP(0, "Attribute:[%d]%s", Attribute, sql_stmt_attr_type(Attribute)); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN doSQLExecute(SQLHSTMT StatementHandle) +{ + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(!STMT_IS_EXECUTING(stmt), ""); + stmt_close_rs(stmt); + OILE(STMT_IS_NORM(stmt), ""); + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + return stmt_execute(stmt); + + // sql_t *sql = (sql_t*)StatementHandle; + // OILE(sql); + // OILE(IS_PREPARED(sql)); + // OILE(!IS_EXECUTING(sql)); + + // rs_t *rs = &sql->rs; + // OILE(rs->taos_rs.rs_is_opened_by_taos_query==0); + // rs_close(rs); + // OILE(rs->sql==NULL); + + // SET_EXECUTING(sql); + + // paramset_t *paramset = &sql->paramset; + // OILE(paramset); + + // // fetch parameters + // OILE(paramset->n_bps==paramset->n_params); + // if (paramset->paramset_size==0) { + // // nodejs workaround + // paramset->paramset_size = 1; + // } + // for (unsigned int i=0; iparamset_size; ++i) { + // for (unsigned int j=0; jn_bps && jn_params; ++j) { + // SQLRETURN r = sql_process_param(sql, i, j); + // if (r!=SQL_SUCCESS) return r; + // } + // OILE(paramset->taos_binds); + // int tr = taos_stmt_bind_param(sql->stmt, paramset->taos_binds); + // if (tr) { + // SET_ERROR(sql, "HY000", "failed to bind parameters[%d in total][%d]%s", paramset->n_params, tr, tstrerror(tr)); + // // keep executing/executed state unchanged + // return SQL_ERROR; + // } + + // tr = taos_stmt_add_batch(sql->stmt); + // if (tr) { + // SET_ERROR(sql, "HY000", "failed to add batch:[%d]%s", tr, tstrerror(tr)); + // // keep executing/executed state unchanged + // return SQL_ERROR; + // } + // } + + // if (1) { + // int r = taos_stmt_execute(sql->stmt); + // if (r) { + // SET_ERROR(sql, "HY000", "failed to execute statement:[%d]%s", r, tstrerror(r)); + // // keep executing/executed state unchanged + // return SQL_ERROR; + // } + + // taos_rs_t *taos_rs = &rs->taos_rs; + // OILE(taos_rs->owner==NULL); + // taos_rs->owner = rs; + // taos_rs->rs = taos_stmt_use_result(sql->stmt); + // } + + // SET_EXECUTED(sql); + + // return sql_after_exec(sql); +} + +static SQLRETURN doSQLFetch(SQLHSTMT StatementHandle) +{ + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(STMT_IS_EXECUTED(stmt), ""); + + if (STMT_TYPEINFO(stmt)) { + return SQL_SUCCESS; + } + + return stmt_fetch(stmt); + // sql_t *sql = (sql_t*)StatementHandle; + // OILE(sql); + // rs_t *rs = &sql->rs; + // OILE(rs->n_rows==-1 || rs->curr_rown_rows); + // row_t *row = &rs->row; + // taos_rs_t *taos_rs = &rs->taos_rs; + // OA(rs->n_fields==taos_rs->n_fields, "%d/%d", rs->n_fields, taos_rs->n_fields); + + // if (rs->eof) return SQL_NO_DATA; + + // if (rs->n_rows!=-1 && rs->curr_row + 1 >= rs->n_rows) { + // row_release(row); + // taos_rs->row = NULL; + // rs->eof = 1; + // return SQL_NO_DATA; + // } + + // row_release(row); + // taos_rs->row = NULL; + // ++rs->curr_row; + // row->valid = 1; + // taos_rs->row = taos_fetch_row(taos_rs->rs); + // D("row:[%p]", taos_rs->row); + // if (!taos_rs->row) { + // row->valid = 0; + // rs->eof = 1; + // D("..."); + // return SQL_NO_DATA; + // } + + // // column bound? + // if (!rs->bcs) return SQL_SUCCESS; + // for (int i=0; in_fields; ++i) { + // SQLRETURN r = sql_get_data(sql, (unsigned int)i); + // if (r!=SQL_SUCCESS) return r; + // } + + // return SQL_SUCCESS; +} + +static SQLRETURN doSQLGetData(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, + SQLPOINTER TargetValue, SQLLEN BufferLength, + SQLLEN *StrLen_or_IndPtr) +{ + P("ColumnNumber:[%d]; TargetType:[%d]%s; BufferLength:[%ld]; StrLen_or_IndPtr:[%p]", + ColumnNumber, + TargetType, sql_c_type(TargetType), + BufferLength, StrLen_or_IndPtr); + + OILE(TargetValue, ""); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(STMT_IS_EXECUTED(stmt), ""); + + if (STMT_TYPEINFO(stmt)) { + ONIY(ColumnNumber==3, ""); + ONIY(TargetType==SQL_C_LONG, ""); + int32_t v = 255; + switch (stmt->typeinfo) { + case SQL_VARCHAR: + case SQL_WVARCHAR: + case SQL_VARBINARY: + break; + case SQL_TIMESTAMP: + case SQL_TYPE_TIMESTAMP: + v = 23; + break; + default: + { + OILE(0, ""); + } break; + } + *(int32_t*)TargetValue = v; + return SQL_SUCCESS; + } + + errs_t *errs = &stmt->errs; + + OILE(ColumnNumber>=0, ""); + if (ColumnNumber==0) { + SET_ERR(errs, "07009", "invalid ColumnNumber[%d]", ColumnNumber); + return SQL_ERROR; + } + + col_binding_t binding = { + .ColumnNumber = ColumnNumber, + .TargetType = TargetType, + .TargetValue = TargetValue, + .BufferLength = BufferLength, + .StrLen_or_IndPtr = StrLen_or_IndPtr + }; + + return stmt_get_data(stmt, &binding); + + // ONIY(TargetValue); + + // sql_t *sql = (sql_t*)StatementHandle; + // OILE(sql); + // rs_t *rs = &sql->rs; + // OILE(rs->sql==sql); + + // if (rs->eof) return SQL_NO_DATA; + // OILE(rs->curr_row>=0); + // OILE(rs->n_rows==-1 || rs->curr_rown_rows); + + // field_t *fields = rs->fields; + // if (!fields) return SQL_NO_DATA; + + // row_t *row = &rs->row; + // OILE(row->valid); + + // col_t *cols = row->cols; + + // if (cols==cols_timestamp || cols==cols_varchar || cols==cols_wvarchar || cols==cols_varbinary) { + // ONIY(ColumnNumber==3); + // ONIY(ColumnNumber<=row->n_cols); + // col_t *col = cols + (ColumnNumber-1); + // OILE(col->valid); + // ONIY(col->c_type==TargetType); + // OILE(col->c_type==SQL_C_LONG); + // int32_t v = col->u.c_long; + // ONIY(BufferLength>=sizeof(v)); + // ONIY(StrLen_or_IndPtr==0); + // *(int32_t*)TargetValue = v; + // return SQL_SUCCESS; + // } + + // OILE(StrLen_or_IndPtr); + // *StrLen_or_IndPtr = SQL_NULL_DATA; + + // return SQL_SUCCESS; +} + +static SQLRETURN doSQLGetDiagField(SQLSMALLINT HandleType, SQLHANDLE Handle, SQLSMALLINT RecNumber, SQLSMALLINT DiagIdentifier, + SQLPOINTER DiagInfo, SQLSMALLINT BufferLength, SQLSMALLINT *StringLength) +{ + P("HandleType:[%d]%s; RecNumber:[%d]; DiagIdentifier:[%d]%s; BufferLength:[%d]", + HandleType, sql_handle_type(HandleType), RecNumber, + DiagIdentifier, sql_diag_identifier(DiagIdentifier), + BufferLength); + OILE(0, ""); + // terror_t *err = NULL; + // switch (HandleType) + // { + // case SQL_HANDLE_ENV: + // { + // env_t *env = (env_t*)Handle; + // err = &env->err; + // } break; + // case SQL_HANDLE_DBC: + // { + // conn_t *conn = (conn_t*)Handle; + // err = &conn->err; + // } break; + // case SQL_HANDLE_STMT: + // { + // sql_t *sql = (sql_t*)Handle; + // err = &sql->err; + // } break; + // default: + // { + // OILE(0); + // return SQL_ERROR; + // } break; + // } + + // OILE(err); + // return SQL_NO_DATA; +} + +static SQLRETURN doSQLGetTypeInfo(SQLHSTMT StatementHandle, SQLSMALLINT DataType) +{ + P("DataType:[%d]%s", DataType, sql_sql_type(DataType)); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(!STMT_IS_EXECUTING(stmt), ""); + stmt_close_rs(stmt); + OILE(STMT_IS_NORM(stmt), ""); + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + switch (DataType) + { + case SQL_VARCHAR: + case SQL_WVARCHAR: + case SQL_VARBINARY: + case SQL_TIMESTAMP: + case SQL_TYPE_TIMESTAMP: + { + stmt->typeinfo = DataType; + } break; + default: + { + ONIY(0, ""); + return SQL_ERROR; + } break; + } + + STMT_SET_EXECUTED(stmt); + + return SQL_SUCCESS; +} + +static SQLRETURN doSQLDescribeParam( + SQLHSTMT StatementHandle, + SQLUSMALLINT ParameterNumber, + SQLSMALLINT * DataTypePtr, + SQLULEN * ParameterSizePtr, + SQLSMALLINT * DecimalDigitsPtr, + SQLSMALLINT * NullablePtr) +{ + // SQLRETURN r; + + OILE(ParameterNumber>0, ""); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(stmt->prepared, ""); + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + errs_t *errs = &stmt->errs; + + // const char *enc = conn->enc_char; + + paramset_t *paramset = &stmt->paramset; + param_t *params = paramset->params; + int n_params = paramset->n_params; + + if (ParameterNumber>n_params) { + SET_ERR(errs, "07009", "invalid ParameterNumber/#params [%d/%d]", ParameterNumber, n_params); + return SQL_ERROR; + } + + param_t *param = params + (ParameterNumber-1); + if (DataTypePtr) *DataTypePtr = param->DataType; + if (ParameterSizePtr) *ParameterSizePtr = param->ParameterSize; + if (DecimalDigitsPtr) *DecimalDigitsPtr = param->DecimalDigits; + if (NullablePtr) *NullablePtr = param->Nullable; + + return SQL_SUCCESS; + + // sql_t *sql = (sql_t*)StatementHandle; + // rs_t *rs = &sql->rs; + // OILE(rs->sql==NULL); + // OILE(IS_PREPARED(sql)); + // OILE(DataTypePtr); + // OILE(ParameterSizePtr); + // OILE(DecimalDigitsPtr); + + // paramset_t *paramset = &sql->paramset; + // OILE(paramset->sql); + // OILE(ParameterNumber>0 && ParameterNumber<=paramset->n_params); + + // *DataTypePtr = SQL_C_CHAR; + // *ParameterSizePtr = 23; + // *DecimalDigitsPtr = 0; + // if (NullablePtr) *NullablePtr = SQL_NULLABLE; + + // param_t *param = paramset->params + (ParameterNumber-1); + // int taos_type = param->taos_type; + // switch (taos_type) + // { + // case TSDB_DATA_TYPE_TIMESTAMP: + // { + // *DataTypePtr = SQL_CHAR; + // *ParameterSizePtr = 23; + // *DecimalDigitsPtr = 0; + // if (NullablePtr) *NullablePtr = SQL_NULLABLE; + // } break; + // case TSDB_DATA_TYPE_TINYINT: + // { + // *DataTypePtr = SQL_TINYINT; + // *ParameterSizePtr = 1; + // *DecimalDigitsPtr = 0; + // if (NullablePtr) *NullablePtr = SQL_NULLABLE; + // } break; + // default: + // { + // OA(0, "taos param:[%d][%d]%s", ParameterNumber, taos_type, taos_data_type(taos_type)); + // return SQL_ERROR; + // } break; + // } + + // P("ParameterNumber:[%d]; DataTypePtr:[%p]%s; ParameterSizePtr:[%p]%ld; DecimalDigitsPtr:[%p]%d; NullablePtr:[%p]%d", + // ParameterNumber, + // DataTypePtr, DataTypePtr ? sql_sql_type(*DataTypePtr) : "UNKNOWN", + // ParameterSizePtr, ParameterSizePtr ? *ParameterSizePtr : 0, + // DecimalDigitsPtr, DecimalDigitsPtr ? *DecimalDigitsPtr : 0, + // NullablePtr, NullablePtr ? *NullablePtr : 0); + // return SQL_SUCCESS; +} + +static SQLRETURN doSQLFreeConnect(SQLHDBC ConnectionHandle) +{ + conn_t *conn = (conn_t*)ConnectionHandle; + OILE(conn->stmts.count==0, ""); + conn_free(conn); + return SQL_SUCCESS; +} + +static SQLRETURN doSQLFreeEnv(SQLHENV EnvironmentHandle) +{ + env_t *env = (env_t*)EnvironmentHandle; + env_dec_ref(env); + return SQL_SUCCESS; +} + +static SQLRETURN doSQLBindCol(SQLHSTMT StatementHandle, + SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, + SQLPOINTER TargetValue, + SQLLEN BufferLength, SQLLEN *StrLen_or_IndPtr) +{ + P("ColumnNumber:[%d]; TargetType:[%d]%s; BufferLength:[%ld]", + ColumnNumber, TargetType, sql_c_type(TargetType), BufferLength); + + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + OILE(STMT_IS_EXECUTED(stmt), ""); + + errs_t *errs = &stmt->errs; + + OILE(ColumnNumber>=0, ""); + if (ColumnNumber==0) { + SET_ERR(errs, "07009", "invalid ColumnNumber[%d]", ColumnNumber); + return SQL_ERROR; + } + + col_binding_t binding = { + .ColumnNumber = ColumnNumber, + .TargetType = TargetType, + .TargetValue = TargetValue, + .BufferLength = BufferLength, + .StrLen_or_IndPtr = StrLen_or_IndPtr + }; + + return stmt_bind_col(stmt, &binding); + + + // OILE(ColumnNumber>0); + // sql_t *sql = (sql_t*)StatementHandle; + // OILE(!IS_EXECUTING(sql)); + + // rs_t *rs = &sql->rs; + // OILE(rs->fields); + // OILE(rs->n_fields>=ColumnNumber); + + // if (!rs->bcs_cache) { + // rs->bcs_cache = todbc_buf_create(); + // if (!rs->bcs_cache) { + // SET_ERROR(sql, "HY001", ""); + // return SQL_ERROR; + // } + // OILE(rs->bcs==NULL); + // rs->bcs = (bc_t*)todbc_buf_calloc(rs->bcs_cache, (size_t)rs->n_fields, sizeof(*rs->bcs)); + // if (!rs->bcs) { + // SET_ERROR(sql, "HY001", ""); + // return SQL_ERROR; + // } + // } + // OILE(rs->bcs); + + // bc_t *bc = rs->bcs + (ColumnNumber-1); + // bc->ColumnNumber = ColumnNumber; + // bc->TargetType = TargetType; + // bc->TargetValue = TargetValue; + // bc->BufferLength = BufferLength; + // bc->StrLen_or_IndPtr = StrLen_or_IndPtr; + + // return SQL_SUCCESS; +} + +static SQLRETURN doSQLCloseCursor(SQLHSTMT StatementHandle) +{ + stmt_t *stmt = (stmt_t*)StatementHandle; + OILE(stmt, ""); + errs_t *errs = &stmt->errs; + + if (STMT_IS_NORM(stmt)) { + OW("no cursor was opened previously"); + SET_ERR(errs, "24000", ""); + return SQL_ERROR; + } + + OILE(STMT_IS_EXECUTED(stmt), ""); + stmt_close_rs(stmt); + + return SQL_SUCCESS; + + // sql_t *sql = (sql_t*)StatementHandle; + // OILE(!IS_EXECUTING(sql)); + + // rs_t *rs = &sql->rs; + // if (!rs->sql) { + // SET_ERROR(sql, "24000", ""); + // OW("no cursor was opened previously"); + // return SQL_ERROR; + // } + + // OILE(rs->sql==sql); + // rs_close(rs); + + // return SQL_SUCCESS; +} + +static SQLRETURN doSQLError(SQLHENV EnvironmentHandle, + SQLHDBC ConnectionHandle, SQLHSTMT StatementHandle, + SQLCHAR *Sqlstate, SQLINTEGER *NativeError, + SQLCHAR *MessageText, SQLSMALLINT BufferLength, + SQLSMALLINT *TextLength) +{ + env_t *env = (env_t*)EnvironmentHandle; + conn_t *conn = (conn_t*)ConnectionHandle; + stmt_t *stmt = (stmt_t*)StatementHandle; + OD("env/conn/stmt:[%p/%p/%p]", env, conn, stmt); + + SQLSMALLINT HandleType; + SQLHANDLE Handle; + if (stmt) { + HandleType = SQL_HANDLE_STMT; + Handle = StatementHandle; + } else if (conn) { + HandleType = SQL_HANDLE_DBC; + Handle = ConnectionHandle; + } else if (env) { + HandleType = SQL_HANDLE_ENV; + Handle = EnvironmentHandle; + } else { + return SQL_NO_DATA; + } + + return doSQLGetDiagRec(HandleType, Handle, 1, Sqlstate, NativeError, MessageText, BufferLength, TextLength); +} + + +#ifdef _MSC_VER + +#define POST_INSTALLER_ERROR(hwndParent, code, fmt, ...) \ +do { \ + char buf[4096]; \ + snprintf(buf, sizeof(buf), "%s[%d]%s():" fmt "", \ + basename((char*)__FILE__), __LINE__, __func__, \ + ##__VA_ARGS__); \ + SQLPostInstallerError(code, buf); \ + if (hwndParent) { \ + MessageBox(hwndParent, buf, "Error", MB_OK|MB_ICONEXCLAMATION); \ + } \ +} while (0) + +typedef struct kv_s kv_t; +struct kv_s { + char *line; + size_t val; +}; + +static BOOL get_driver_dll_path(HWND hwndParent, char *buf, size_t len) +{ + HMODULE hm = NULL; + + if (GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCSTR) &ConfigDSN, &hm) == 0) + { + int ret = GetLastError(); + POST_INSTALLER_ERROR(hwndParent, ODBC_ERROR_REQUEST_FAILED, "GetModuleHandle failed, error = %d\n", ret); + return FALSE; + } + if (GetModuleFileName(hm, buf, (DWORD)len) == 0) + { + int ret = GetLastError(); + POST_INSTALLER_ERROR(hwndParent, ODBC_ERROR_REQUEST_FAILED, "GetModuleFileName failed, error = %d\n", ret); + return FALSE; + } + return TRUE; +} + +static BOOL doDSNAdd(HWND hwndParent, LPCSTR lpszDriver, LPCSTR lpszAttributes) +{ + BOOL r = TRUE; + + kv_t *kvs = NULL; + + kv_t dsn = {0}; + char *line = NULL; + + do { + char driver_dll[MAX_PATH + 1]; + r = get_driver_dll_path(hwndParent, driver_dll, sizeof(driver_dll)); + if (!r) break; + + dsn.line = strdup("DSN=TAOS_DEMO"); + if (!dsn.line) { r = FALSE; break; } + + const char *p = lpszAttributes; + int ikvs = 0; + while (p && *p) { + line = strdup(p); + if (!line) { r = FALSE; break; } + char *v = strchr(line, '='); + if (!v) { r = FALSE; break; } + + if (strstr(line, "DSN")==line) { + if (dsn.line) { + free(dsn.line); + dsn.line = NULL; + dsn.val = 0; + } + dsn.line = line; + line = NULL; + } else { + kv_t *t = (kv_t*)realloc(kvs, (ikvs+1)*sizeof(*t)); + if (!t) { r = FALSE; free(line); break; } + t[ikvs].line = line; + *v = '\0'; + if (v) t[ikvs].val = v - line + 1; + line = NULL; + + kvs = t; + ++ikvs; + } + + p += strlen(p) + 1; + } + + if (hwndParent) { + MessageBox(hwndParent, "Please use odbcconf to add DSN for TAOS ODBC Driver", "Warning!", MB_OK|MB_ICONEXCLAMATION); + } + if (!r) break; + + char *v = NULL; + v = strchr(dsn.line, '='); + if (!v) { r = FALSE; break; } + *v = '\0'; + dsn.val = v - dsn.line + 1; + + if ((!dsn.line)) { + if (!r) POST_INSTALLER_ERROR(hwndParent, ODBC_ERROR_REQUEST_FAILED, "lack of either DSN or Driver"); + } else { + if (r) r = SQLWritePrivateProfileString("ODBC Data Sources", dsn.line+dsn.val, lpszDriver, "Odbc.ini"); + if (r) r = SQLWritePrivateProfileString(dsn.line+dsn.val, "Driver", driver_dll, "Odbc.ini"); + } + + for (int i=0; r && i + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _base_h_ +#define _base_h_ + +#include "todbc_buf.h" +#include "todbc_iconv.h" +#include "todbc_log.h" + +#include "taos.h" +#include "taoserror.h" + +#include +#include + +typedef struct errs_s errs_t; +typedef struct env_s env_t; +typedef struct conn_s conn_t; +typedef struct stmt_s stmt_t; +typedef struct param_s param_t; +typedef struct field_s field_t; +typedef struct rs_s rs_t; +typedef struct col_s col_t; + +#define GET_REF(obj) atomic_load_64(&obj->refcount) +#define INC_REF(obj) atomic_add_fetch_64(&obj->refcount, 1) +#define DEC_REF(obj) atomic_sub_fetch_64(&obj->refcount, 1) + +// public +#ifdef __GNUC__ + __attribute__((format(printf, 6, 7))) +#endif +SQLRETURN errs_append(errs_t *errs, const char sql_state[6], const char *file, int line, const char *func, const char *fmt, ...); +int errs_count(errs_t *errs); +// 0/-1: ok/no-error +int errs_fetch(errs_t *errs, int idx, const char **sql_state, const char **err_str); +void errs_clear(SQLSMALLINT HandleType, SQLHANDLE InputHandle); + +// err: if <>0, will generate strerror +#define SET_ERR(errs, sql_state, fmt, ...) \ + errs_append(errs, sql_state, __FILE__, __LINE__, __func__, "%s" fmt "", "", ##__VA_ARGS__) + +#define SET_OOM(errs, fmt, ...) SET_ERR(errs, "TD001", "OOM:" fmt, ##__VA_ARGS__) +#define SET_NIY(errs, fmt, ...) SET_ERR(errs, "TDC00", "NIY:" fmt, ##__VA_ARGS__) +#define SET_GENERAL(errs, fmt, ...) SET_ERR(errs, "TD000", "GEN:" fmt, ##__VA_ARGS__) + + +// public +errs_t* env_get_errs(env_t *env); +void env_clr_errs(env_t *env); +void env_inc_ref(env_t *env); +void env_dec_ref(env_t *env); + +// public +errs_t* conn_get_errs(conn_t *conn); +void conn_clr_errs(conn_t *conn); + +// public +errs_t* stmt_get_errs(stmt_t *stmt); +void stmt_clr_errs(stmt_t *stmt); + +#endif // _base_h_ + + diff --git a/src/connector/odbc/src/base/CMakeLists.txt b/src/connector/odbc/src/base/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..fa13f3e07737bb6c2a36e5296a49a9f282346e3b --- /dev/null +++ b/src/connector/odbc/src/base/CMakeLists.txt @@ -0,0 +1,10 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +aux_source_directory(. SRC) +add_library(todbc_base STATIC ${SRC}) + +if (TD_DARWIN) + target_include_directories(todbc_base PRIVATE /usr/local/include) +endif () + diff --git a/src/connector/odbc/src/base/conn.c b/src/connector/odbc/src/base/conn.c new file mode 100644 index 0000000000000000000000000000000000000000..6e0554d88552330e2589eceb419a06d2135b54ce --- /dev/null +++ b/src/connector/odbc/src/base/conn.c @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "conn.h" + +#include "env.h" + +#include "../todbc_tls.h" + +static void init_encodes(conn_t *conn, env_t *env) { + OILE(conn, ""); + OILE(env, ""); + const char *enc_charset = env->enc_charset; + + snprintf(conn->enc_src, sizeof(conn->enc_src), "%s", UTF8_ENC); // compile-time constant + snprintf(conn->enc_wchar, sizeof(conn->enc_wchar), "%s", UTF16_ENC); // compile-time constant + snprintf(conn->enc_char, sizeof(conn->enc_char), "%s", enc_charset); // runtime default + snprintf(conn->enc_db, sizeof(conn->enc_db), "%s", enc_charset); // runtime default + snprintf(conn->enc_locale, sizeof(conn->enc_locale), "%s", enc_charset); // runtime default + + OD("enc_src:[%s]; enc_wchar:[%s]; enc_char:[%s]; enc_db:[%s]; enc_locale:[%s]", + conn->enc_src, conn->enc_wchar, conn->enc_char, conn->enc_db, conn->enc_locale); +} + +static int conn_init(conn_t *conn, env_t *env) { + OILE(conn, ""); + OILE(env, ""); + errs_t *errs = &env->errs; + + OILE(conn->env==NULL, ""); + + int r = errs_init(&conn->errs); + if (r) return -1; + + init_encodes(conn, env); + if (SQL_SUCCESS!=conn_check_charset(conn, errs)) { + return -1; + } + + conn->env = env; + env_inc_ref(env); + + return 0; +}; + +static void conn_release(conn_t *conn) { + if (!conn) return; + env_t *env = conn->env; + if (!env) return; + + conn->env = NULL; + env_dec_ref(env); +} + +conn_t* conn_new(env_t *env) { + conn_t *conn = (conn_t*)calloc(1, sizeof(*conn)); + if (!conn) return NULL; + + if (conn_init(conn, env)) { + OILE(conn->env==NULL, ""); + free(conn); + return NULL; + } + + return conn; +} + +void conn_free(conn_t *conn) { + if (!conn) return; + + // clean ext stuff + if (conn->ext.free_conn) { + conn->ext.free_conn(conn); + } + + // clean conn stuffs + conn_release(conn); + + free(conn); +} + +static SQLRETURN do_check_charset(errs_t *errs, const char *enc_charset, todbc_enc_t *enc) { + *enc = todbc_tls_iconv_enc(enc_charset); + if (enc->enc[0]=='\0') { + if (errs) { + SET_GENERAL(errs, "unknown charset [%s]", enc_charset); + } + return SQL_ERROR; + } + + return SQL_SUCCESS; +} + +SQLRETURN conn_check_charset(conn_t *conn, errs_t *errs) { + OILE(conn, ""); + + todbc_enc_t enc; + + SQLRETURN r; + r = do_check_charset(errs, conn->enc_char, &enc); + if (r!=SQL_SUCCESS) return r; + r = do_check_charset(errs, conn->enc_db, &enc); + if (r!=SQL_SUCCESS) return r; + r = do_check_charset(errs, conn->enc_locale, &enc); + if (r!=SQL_SUCCESS) return r; + r = do_check_charset(errs, conn->enc_src, &enc); + if (r!=SQL_SUCCESS) return r; + + r = do_check_charset(errs, conn->enc_wchar, &enc); + if (r!=SQL_SUCCESS) return r; + + if (enc.variable_char_size!=-1) { + OE("does not support [%s] for WCHAR", conn->enc_wchar); + if (errs) { + SET_GENERAL(errs, "does not support [%s] for WCHAR", conn->enc_wchar); + } + return SQL_ERROR; + } + if (enc.char_size<=0) { + if (errs) { + SET_GENERAL(errs, "unknown [%s] for WCHAR", conn->enc_wchar); + } + return SQL_ERROR; + } + + conn->wchar_size = (size_t)enc.char_size; + + return SQL_SUCCESS; +} + +int conn_add_stmt(conn_t *conn, stmt_t *stmt) { + OILE(conn, ""); + OILE(stmt, ""); + OILE(stmt->owner==NULL, ""); + OILE(stmt->next==NULL, ""); + OILE(stmt->prev==NULL, ""); + OILE(conn->ext.init_stmt, ""); + + if (conn->ext.init_stmt(stmt)) { + return -1; + } + + stmts_t *owner = &conn->stmts; + + stmt->owner = owner; + + stmt->prev = owner->tail; + if (owner->tail) owner->tail->next = stmt; + else owner->head = stmt; + owner->tail = stmt; + + ++owner->count; + owner->conn = conn; + + return 0; +} + +void conn_del_stmt(conn_t *conn, stmt_t *stmt) { + OILE(conn, ""); + OILE(stmt, ""); + OILE(stmt->owner, ""); + OILE(stmt->owner==&conn->stmts, ""); + OILE(stmt->owner->conn==conn, ""); + + stmts_t *owner = stmt->owner; + + stmt_t *next = stmt->next; + stmt_t *prev = stmt->prev; + + if (next) next->prev = prev; + else owner->tail = prev; + + if (prev) prev->next = next; + else owner->head = next; + + --owner->count; + + stmt->next = NULL; + stmt->prev = NULL; + stmt->owner = NULL; +} + +SQLRETURN conn_connect(conn_t *conn) { + OILE(conn, ""); + OILE(conn->ext.connect, ""); + return conn->ext.connect(conn); +} + +void conn_disconnect(conn_t *conn) { + OILE(conn, ""); + OILE(conn->ext.disconnect, ""); + conn->ext.disconnect(conn); +} + +// public +errs_t* conn_get_errs(conn_t *conn) { + OILE(conn, ""); + + return &conn->errs; +} + +void conn_clr_errs(conn_t *conn) { + if (!conn) return; + + errs_reclaim(&conn->errs); +} + + + + + diff --git a/src/connector/odbc/src/base/conn.h b/src/connector/odbc/src/base/conn.h new file mode 100644 index 0000000000000000000000000000000000000000..d82cbc37d4bb6a5a4c0f82e13396fa0f5342147b --- /dev/null +++ b/src/connector/odbc/src/base/conn.h @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _conn_h_ +#define _conn_h_ + +#include "../base.h" + +#include "stmt.h" + + +#include "../todbc_flex.h" + +typedef struct conn_ext_s conn_ext_t; +struct conn_ext_s { + void *ext; + void (*free_conn)(conn_t *conn); + int (*init_stmt)(stmt_t *stmt); + + SQLRETURN (*connect)(conn_t *conn); + void (*disconnect)(conn_t *conn); +}; + +struct conn_s { + env_t *env; + + char enc_src[64]; // c source file encoding + char enc_char[64]; // SQL_CHAR encoding + char enc_wchar[64]; // SQL_WCHAR encoding + char enc_db[64]; // taos client encoding + // use this for system i/o, such as reading from stdin, writing to stdout/stderr + char enc_locale[64]; // default: current localee + + size_t wchar_size; // shall be fix-length + + conn_val_t val; + + stmts_t stmts; + + errs_t errs; + + conn_ext_t ext; + + unsigned int connect:2; +}; + +#define CONN_SET_CONNECTING(conn) (conn->connect=0x01) +#define CONN_SET_CONNECTED(conn) (conn->connect=0x02) +#define CONN_SET_NORM(conn) (conn->connect=0x00) + +#define CONN_IS_CONNECTING(conn) (conn->connect==0x01) +#define CONN_IS_CONNECTED(conn) (conn->connect==0x02) +#define CONN_IS_NORM(conn) (conn->connect==0x00) + +conn_t* conn_new(env_t *env); +void conn_free(conn_t *conn); + +SQLRETURN conn_check_charset(conn_t *conn, errs_t *errs); + +int conn_add_stmt(conn_t *conn, stmt_t *stmt); +void conn_del_stmt(conn_t *conn, stmt_t *stmt); + +SQLRETURN conn_connect(conn_t *conn); +void conn_disconnect(conn_t *conn); + + +#endif // _conn_h_ + + diff --git a/src/connector/odbc/src/base/env.c b/src/connector/odbc/src/base/env.c new file mode 100644 index 0000000000000000000000000000000000000000..75b5b8360209f221b69b66e34c58cb7da439affe --- /dev/null +++ b/src/connector/odbc/src/base/env.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "env.h" + +#include + +static pthread_once_t init_once = PTHREAD_ONCE_INIT; +static char default_charset[64] = {0}; + +static void init_routine(void) { + OD("compiled with ODBCVER:[0x%04x]", ODBCVER); + + const char *charset = NULL; + setlocale(LC_ALL, ""); + const char *locale = setlocale(LC_CTYPE, NULL); + if (locale) { + const char *dot = strrchr(locale, '.'); + if (dot) charset = dot + 1; + } + if (!charset) { +#ifdef _MSC_VER + charset = "CP936"; +#else + charset = "UTF-8"; +#endif + OD("failed to find original locale, fall back to [%s]", charset); + } else { +#ifdef _MSC_VER + char buf[64]; + snprintf(buf, sizeof(buf), "CP%s", charset); + charset = buf; +#endif + OD("system default charset: [%s]", charset); + } + + snprintf(default_charset, sizeof(default_charset), "%s", charset); +} + + +static void env_release(env_t *env) { + if (!env) return; + OILE(env->refcount==0, ""); + + env_clr_errs(env); +} + +int env_init(env_t *env) { + OILE(env, ""); + OILE(env->refcount==0, ""); + + pthread_once(&init_once, init_routine); + + int r = errs_init(&env->errs); + if (r) return -1; + + snprintf(env->enc_charset, sizeof(env->enc_charset), "%s", default_charset); + env->odbc_ver = SQL_OV_ODBC3; + + env->refcount = 1; + + return 0; +} + +// public +errs_t* env_get_errs(env_t *env) { + OILE(env, ""); + + return &env->errs; +} + +void env_clr_errs(env_t *env) { + if (!env) return; + + errs_reclaim(&env->errs); +} + +void env_inc_ref(env_t *env) { + OILE(env, ""); + int64_t rc = INC_REF(env); + OILE(rc>=2, ""); +} + +void env_dec_ref(env_t *env) { + OILE(env, ""); + int64_t rc = DEC_REF(env); + if (rc>0) return; + OILE(rc==0, ""); + + env_release(env); + free(env); +} + +env_t* env_create(void) { + env_t *env = (env_t*)calloc(1, sizeof(*env)); + if (!env) return NULL; + + if (env_init(env)) { + free(env); + return NULL; + } + + return env; +} + diff --git a/src/connector/odbc/src/base/env.h b/src/connector/odbc/src/base/env.h new file mode 100644 index 0000000000000000000000000000000000000000..b4f420ad64d541fac92c8a4105e1d89b1cd0e9a0 --- /dev/null +++ b/src/connector/odbc/src/base/env.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _env_h_ +#define _env_h_ + +#include "../base.h" + +#include "err.h" + +struct env_s { + int64_t refcount; + + char enc_charset[64]; // default charset from system locale + int32_t odbc_ver; // default SQL_OV_ODBC3 + + errs_t errs; + + void (*env_free)(env_t* env); +}; + +int env_init(env_t *env); + +env_t* env_create(void); + + + + +#endif // _env_h_ + + diff --git a/src/connector/odbc/src/base/err.c b/src/connector/odbc/src/base/err.c new file mode 100644 index 0000000000000000000000000000000000000000..c1547288076fa5c4e53099560ae211ce14f61b65 --- /dev/null +++ b/src/connector/odbc/src/base/err.c @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "err.h" + +#include "env.h" +#include "conn.h" +#include "stmt.h" + +struct err_s { + char sql_state[6]; + char *err_str; // no ownership +}; + +static void errs_clr(errs_t *errs) { + if (!errs) return; + + errs->count = 0; + errs->errs = NULL; +} + +int errs_init(errs_t *errs) { + OILE(errs && errs->cache==NULL, ""); + OILE(errs->count==0 && errs->errs==NULL, ""); + + errs->cache = todbc_buf_create(); + + return errs->cache ? 0 : -1; +} + +void errs_reclaim(errs_t *errs) { + if (!errs) return; + if (!errs->cache) return; + + errs_clr(errs); + + todbc_buf_reclaim(errs->cache); +} + +void errs_release(errs_t *errs) { + if (!errs) return; + if (!errs->cache) return; + + errs_clr(errs); + + todbc_buf_free(errs->cache); + errs->cache = NULL; +} + +// public +#ifdef __GNUC__ + __attribute__((format(printf, 6, 7))) +#endif +SQLRETURN errs_append(errs_t *errs, const char sql_state[6], const char *file, int line, const char *func, const char *fmt, ...) +{ + OILE(errs, ""); + OILE(errs->cache, ""); + todbc_buf_t *cache = errs->cache; + + const char *name = basename((char*)file); + + char *buf = NULL; + size_t blen = 0; + while (1) { + char *p = buf; + size_t bytes = blen; + + int count = 0; + int n = 0; + + va_list ap; + va_start(ap, fmt); + if (bytes<0) bytes = 0; + n = vsnprintf(p, bytes, fmt, ap); + va_end(ap); + + OILE(n>=0, ""); + + count += n; + if (p) p += n; + if (bytes) bytes -= (size_t)n; + + if (bytes<0) bytes = 0; + n = snprintf(p, bytes, "@%s[%d]%s()\n", name, line, func); + + OILE(n>=0, ""); + count += n; + + if (p) break; + + buf = todbc_buf_alloc(cache, (size_t)count + 1); + if (!buf) return SQL_ERROR; + blen = (size_t)count; + } + + size_t bytes = (size_t)(errs->count + 1) * sizeof(err_t); + + err_t *es = (err_t*)todbc_buf_realloc(cache, errs->errs, bytes); + if (!es) return SQL_ERROR; + errs->errs = es; + errs->count += 1; + + err_t *err = errs->errs + errs->count - 1; + snprintf(err->sql_state, sizeof(err->sql_state), "%s", sql_state); + err->err_str = buf; + + + return SQL_SUCCESS; +} + +int errs_count(errs_t *errs) { + OILE(errs, ""); + OILE(errs->cache, ""); + + return errs->count; +} + +// 0/-1: ok/no-error +int errs_fetch(errs_t *errs, int idx, const char **sql_state, const char **err_str) { + OILE(errs, ""); + OILE(errs->cache, ""); + + if (errs->count<=0) return -1; + if (idx<0 || idx>=errs->count) return -1; + + err_t *err = errs->errs + idx; + + if (sql_state) *sql_state = err->sql_state; + if (err_str) *err_str = err->err_str; + + return 0; +} + +void errs_clear(SQLSMALLINT HandleType, SQLHANDLE InputHandle) { + errs_t *errs = NULL; + + if (InputHandle==NULL) return; + + switch (HandleType) + { + case SQL_HANDLE_ENV: + { + env_t *env = (env_t*)InputHandle; + errs = &env->errs; + } break; + case SQL_HANDLE_DBC: + { + conn_t *conn = (conn_t*)InputHandle; + errs = &conn->errs; + } break; + case SQL_HANDLE_STMT: + { + stmt_t *stmt = (stmt_t*)InputHandle; + errs = &stmt->errs; + } break; + default: + { + ONIY(0, ""); + } break; + } + + if (!errs) return; + errs_reclaim(errs); +} + diff --git a/src/connector/odbc/src/base/err.h b/src/connector/odbc/src/base/err.h new file mode 100644 index 0000000000000000000000000000000000000000..ae8f9a8085e90acd4f614cdccbfe04f4709ae510 --- /dev/null +++ b/src/connector/odbc/src/base/err.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _err_h_ +#define _err_h_ + +#include "../base.h" + +typedef struct err_s err_t; + +struct errs_s { + todbc_buf_t *cache; + + int count; + err_t *errs; +}; + +int errs_init(errs_t *errs); +void errs_reclaim(errs_t *errs); +void errs_release(errs_t *errs); + +#endif // _err_h_ + diff --git a/src/connector/odbc/src/base/field.c b/src/connector/odbc/src/base/field.c new file mode 100644 index 0000000000000000000000000000000000000000..41a466069fb8f40252c923ac78e3bdfcd864c425 --- /dev/null +++ b/src/connector/odbc/src/base/field.c @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "field.h" + +static void fieldset_clr_fields(fieldset_t *fieldset) { + if (!fieldset) return; + + fieldset->fields = NULL; + fieldset->n_fields = 0; +} + +static void fieldset_clr_bindings(fieldset_t *fieldset) { + if (!fieldset) return; + + fieldset->bindings = NULL; + fieldset->n_bindings = 0; +} + +void fieldset_init_fields(fieldset_t *fieldset) { + if (!fieldset) return; + if (fieldset->fields_cache) return; + fieldset->fields_cache = todbc_buf_create(); +} + +void fieldset_init_bindings(fieldset_t *fieldset) { + if (!fieldset) return; + if (fieldset->bindings_cache) return; + fieldset->bindings_cache = todbc_buf_create(); +} + +void fieldset_reclaim_fields(fieldset_t *fieldset) { + if (!fieldset) return; + + if (!fieldset->fields_cache) return; + + todbc_buf_reclaim(fieldset->fields_cache); + fieldset_clr_fields(fieldset); +} + +void fieldset_reclaim_bindings(fieldset_t *fieldset) { + if (!fieldset) return; + + if (!fieldset->bindings_cache) return; + + todbc_buf_reclaim(fieldset->bindings_cache); + fieldset_clr_bindings(fieldset); +} + +void fieldset_release(fieldset_t *fieldset) { + if (!fieldset) return; + + fieldset_reclaim_fields(fieldset); + fieldset_reclaim_bindings(fieldset); + + if (fieldset->fields_cache) { + todbc_buf_free(fieldset->fields_cache); + fieldset->fields_cache = NULL; + } + + if (fieldset->bindings_cache) { + todbc_buf_free(fieldset->bindings_cache); + fieldset->bindings_cache = NULL; + } +} + diff --git a/src/connector/odbc/src/base/field.h b/src/connector/odbc/src/base/field.h new file mode 100644 index 0000000000000000000000000000000000000000..667dc65ff66e8095e97804ad412adbbd88ed8969 --- /dev/null +++ b/src/connector/odbc/src/base/field.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _field_h_ +#define _field_h_ + +#include "../base.h" + +typedef struct fieldset_s fieldset_t; +typedef struct col_binding_s col_binding_t; +typedef struct field_arg_s field_arg_t; + +// SQLDescribeCol +struct field_arg_s { + SQLUSMALLINT ColumnNumber; + SQLCHAR *ColumnName; + SQLSMALLINT BufferLength; + SQLSMALLINT *NameLength; + SQLSMALLINT *DataType; // sql data type + SQLULEN *ColumnSize; + SQLSMALLINT *DecimalDigits; + SQLSMALLINT *Nullable; +}; + +struct field_s { + SQLUSMALLINT ColumnNumber; + SQLCHAR ColumnName[64]; // hard-coded + SQLSMALLINT DataType; // sql data type + SQLULEN ColumnSize; + SQLSMALLINT DecimalDigits; + SQLSMALLINT Nullable; +}; + +// SQLBindCol; SQLGetData +struct col_binding_s { + SQLUSMALLINT ColumnNumber; + SQLSMALLINT TargetType; // sql c data type + SQLPOINTER TargetValue; + SQLLEN BufferLength; + SQLLEN *StrLen_or_IndPtr; +}; + + +struct fieldset_s { + todbc_buf_t *fields_cache; + field_t *fields; + int n_fields; + + todbc_buf_t *bindings_cache; + col_binding_t *bindings; + int n_bindings; +}; + +void fieldset_init_fields(fieldset_t *fieldset); +void fieldset_init_bindings(fieldset_t *fieldset); + +void fieldset_reclaim_fields(fieldset_t *fieldset); +void fieldset_reclaim_bindings(fieldset_t *fieldset); + +void fieldset_release(fieldset_t *fields); + +#endif // _field_h_ + + + diff --git a/src/connector/odbc/src/base/null_conn.c b/src/connector/odbc/src/base/null_conn.c new file mode 100644 index 0000000000000000000000000000000000000000..29fe86991ac43095bd843a8df961ee2c443ade11 --- /dev/null +++ b/src/connector/odbc/src/base/null_conn.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "null_conn.h" + +#include "env.h" + +// null_conn + +static void null_conn_free(conn_t *conn) { + OILE(conn, ""); + null_conn_t *null_conn = (null_conn_t*)conn->ext.ext; + OILE(null_conn, ""); + + conn->ext.ext = NULL; + conn->ext.free_conn = NULL; + + free(null_conn); +} + +static SQLRETURN null_conn_connect(conn_t *conn) { + OILE(conn, ""); + null_conn_t *null_conn = (null_conn_t*)conn->ext.ext; + OILE(null_conn && null_conn->conn==conn, ""); + OILE(CONN_IS_NORM(conn), ""); + return SQL_SUCCESS; +} + +static void null_conn_disconnect(conn_t *conn) { + OILE(conn, ""); + OILE(CONN_IS_CONNECTED(conn), ""); +} + +static void null_conn_free_stmt(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); +} + +static SQLRETURN null_conn_exec_direct(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + OILE(!STMT_IS_EXECUTED(stmt), ""); + STMT_SET_EXECUTED(stmt); + return SQL_SUCCESS; +} + +static SQLRETURN null_conn_prepare(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + return SQL_SUCCESS; +} + +static SQLRETURN null_conn_proc_param(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + return SQL_SUCCESS; +} + +static SQLRETURN null_conn_execute(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + OILE(!STMT_IS_EXECUTED(stmt), ""); + STMT_SET_EXECUTED(stmt); + return SQL_SUCCESS; +} + +static int null_conn_init_stmt(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->ext.ext==NULL, ""); + OILE(stmt->ext.free_stmt==NULL, ""); + + OILE(stmt->owner==NULL, ""); + + stmt->ext.ext = NULL; + stmt->ext.free_stmt = null_conn_free_stmt; + stmt->ext.exec_direct = null_conn_exec_direct; + stmt->ext.prepare = null_conn_prepare; + stmt->ext.proc_param = null_conn_proc_param; + stmt->ext.execute = null_conn_execute; + + return 0; +} + +int conn_init_null_conn(conn_t *conn) { + OILE(conn, ""); + OILE(conn->ext.ext==NULL, ""); + OILE(conn->ext.free_conn==NULL, ""); + + null_conn_t *null_conn = (null_conn_t*)calloc(1, sizeof(*null_conn)); + if (!null_conn) return -1; + + null_conn->conn = conn; + conn->ext.ext = null_conn; + conn->ext.free_conn = null_conn_free; + conn->ext.connect = null_conn_connect; + conn->ext.disconnect = null_conn_disconnect; + conn->ext.init_stmt = null_conn_init_stmt; + + return 0; +} + + + diff --git a/src/connector/odbc/src/base/null_conn.h b/src/connector/odbc/src/base/null_conn.h new file mode 100644 index 0000000000000000000000000000000000000000..9940cda5848a9e9dbccf866853e154b7e5868d6b --- /dev/null +++ b/src/connector/odbc/src/base/null_conn.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _null_conn_h_ +#define _null_conn_h_ + +#include "../base.h" + +#include "conn.h" + +typedef struct null_conn_s null_conn_t; +struct null_conn_s { + conn_t *conn; +}; + +int conn_init_null_conn(conn_t *conn); + + +#endif // _null_conn_h_ + + diff --git a/src/connector/odbc/src/base/param.c b/src/connector/odbc/src/base/param.c new file mode 100644 index 0000000000000000000000000000000000000000..59b2871f6ed9324e07c5fb23f20b6bcde734644d --- /dev/null +++ b/src/connector/odbc/src/base/param.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "param.h" + +static void paramset_clr_params(paramset_t *paramset) { + if (!paramset) return; + + paramset->params = NULL; + paramset->n_params = 0; + + paramset->i_row = 0; + paramset->i_col = 0; +} + +static void paramset_clr_bindings(paramset_t *paramset) { + if (!paramset) return; + + paramset->bindings = NULL; + paramset->n_bindings = 0; +} + +void paramset_reclaim_params(paramset_t *paramset) { + if (!paramset) return; + + if (!paramset->params_cache) return; + + todbc_buf_reclaim(paramset->params_cache); + paramset_clr_params(paramset); +} + +void paramset_reclaim_bindings(paramset_t *paramset) { + if (!paramset) return; + + if (!paramset->bindings_cache) return; + + todbc_buf_reclaim(paramset->bindings_cache); + paramset_clr_bindings(paramset); +} + +void paramset_init_params_cache(paramset_t *paramset) { + if (!paramset) return; + if (paramset->params_cache) return; + + OILE(paramset->params==NULL, ""); + OILE(paramset->n_params==0, ""); + + paramset->params_cache = todbc_buf_create(); +} + +void paramset_init_bindings_cache(paramset_t *paramset) { + if (!paramset) return; + if (paramset->bindings_cache) return; + + OILE(paramset->bindings==NULL, ""); + OILE(paramset->n_bindings==0, ""); + + paramset->bindings_cache = todbc_buf_create(); +} + +void paramset_release(paramset_t *paramset) { + if (!paramset) return; + + paramset_reclaim_params(paramset); + paramset_reclaim_bindings(paramset); + + if (paramset->params_cache) { + todbc_buf_free(paramset->params_cache); + paramset->params_cache = NULL; + } + + if (paramset->bindings_cache) { + todbc_buf_free(paramset->bindings_cache); + paramset->bindings_cache = NULL; + } +} + diff --git a/src/connector/odbc/src/base/param.h b/src/connector/odbc/src/base/param.h new file mode 100644 index 0000000000000000000000000000000000000000..6175add47b706e927329c78111232aa9617f9b08 --- /dev/null +++ b/src/connector/odbc/src/base/param.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _param_h_ +#define _param_h_ + +#include "../base.h" + +typedef struct paramset_s paramset_t; +typedef struct param_binding_s param_binding_t; +typedef struct param_val_s param_val_t; + +// SQLDescribeParam +struct param_s { + SQLUSMALLINT ParameterNumber; + SQLSMALLINT DataType; // sql data type + SQLULEN ParameterSize; + SQLSMALLINT DecimalDigits; + SQLSMALLINT Nullable; +}; + +// SQLBindParameter +struct param_binding_s { + SQLUSMALLINT ParameterNumber; + SQLSMALLINT InputOutputType; + SQLSMALLINT ValueType; // sql c data type + SQLSMALLINT ParameterType; // sql data type + SQLULEN ColumnSize; + SQLSMALLINT DecimalDigits; + SQLPOINTER ParameterValuePtr; + SQLLEN BufferLength; + SQLLEN *StrLen_or_IndPtr; +}; + +struct paramset_s { + todbc_buf_t *params_cache; + param_t *params; + int n_params; + + todbc_buf_t *bindings_cache; + param_binding_t *bindings; + int n_bindings; + + int i_row; + int i_col; +}; + +void paramset_reclaim_params(paramset_t *paramset); +void paramset_reclaim_bindings(paramset_t *paramset); + +void paramset_init_params_cache(paramset_t *paramset); +void paramset_init_bindings_cache(paramset_t *paramset); + +void paramset_release(paramset_t *paramset); + +#endif // _param_h_ + + + diff --git a/src/connector/odbc/src/base/rs.c b/src/connector/odbc/src/base/rs.c new file mode 100644 index 0000000000000000000000000000000000000000..068cdec4fd00d8e00ee5976ad2a54feb1da10f12 --- /dev/null +++ b/src/connector/odbc/src/base/rs.c @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "rs.h" + + diff --git a/src/connector/odbc/src/base/rs.h b/src/connector/odbc/src/base/rs.h new file mode 100644 index 0000000000000000000000000000000000000000..6a7e0a2b3ac73047456260bce915278fe473fb5b --- /dev/null +++ b/src/connector/odbc/src/base/rs.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _rs_h_ +#define _rs_h_ + +#include "../base.h" + + +struct rs_s { + int affected_rows; +}; + + + +#endif // _rs_h_ + diff --git a/src/connector/odbc/src/base/stmt.c b/src/connector/odbc/src/base/stmt.c new file mode 100644 index 0000000000000000000000000000000000000000..2818b46ed2dbb11594b6c6cb63c61198c558e05f --- /dev/null +++ b/src/connector/odbc/src/base/stmt.c @@ -0,0 +1,465 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "stmt.h" + +#include "conn.h" +#include "param.h" + +static int static_app_row; // SQL_ATTR_APP_ROW_DESC +static int static_app_param; // SQL_ATTR_APP_PARAM_DESC +static int static_imp_row; // SQL_ATTR_IMP_ROW_DESC +static int static_imp_param; // SQL_ATTR_IMP_PARAM_DESC + +static int stmt_init_descs(stmt_t *stmt) { + OILE(stmt, ""); + descs_t *descs = &stmt->descs; + + descs->app_row = &static_app_row; // SQL_ATTR_APP_ROW_DESC + descs->app_param = &static_app_param; // SQL_ATTR_APP_PARAM_DESC + descs->imp_row = &static_imp_row; // SQL_ATTR_IMP_ROW_DESC + descs->imp_param = &static_imp_param; // SQL_ATTR_IMP_PARAM_DESC + + return 0; +} + +static int stmt_init_sql(stmt_t *stmt) { + OILE(stmt, ""); + sql_t *sql = &stmt->sql; + OILE(sql->cache==NULL, ""); + OILE(sql->txt.buf==NULL, ""); + OILE(sql->txt.bytes==0, ""); + OILE(sql->txt.total_bytes==0, ""); + + sql->cache = todbc_buf_create(); + if (!sql->cache) return -1; + + return 0; +} + +static void stmt_release_sql(stmt_t *stmt) { + OILE(stmt, ""); + sql_t *sql = &stmt->sql; + if (!sql->cache) return; + todbc_buf_free(sql->cache); + sql->cache = NULL; + sql->txt.buf = NULL; + sql->txt.bytes = 0; + sql->txt.total_bytes = 0; +} + +static int stmt_init(stmt_t *stmt, conn_t *conn) { + OILE(stmt, ""); + OILE(conn, ""); + OILE(stmt->owner==NULL, ""); + + stmt->typeinfo = SQL_UNKNOWN_TYPE; + + int r = errs_init(&stmt->errs); + if (r) return -1; + + r = stmt_init_descs(stmt); + if (r) return -1; + + r = stmt_init_sql(stmt); + if (r) return -1; + + stmt->attr.bind_type = SQL_PARAM_BIND_BY_COLUMN; + + r = conn_add_stmt(conn, stmt); + OILE(r==0, ""); + OILE(stmt->owner && stmt->owner->conn==conn, ""); + + return 0; +} + +static void stmt_release(stmt_t *stmt) { + if (!stmt) return; + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + + conn_del_stmt(conn, stmt); + + paramset_release(&stmt->paramset); + fieldset_release(&stmt->fieldset); + stmt_release_sql(stmt); + errs_release(&stmt->errs); +} + +static SQLRETURN do_process_param(stmt_t *stmt) { + OILE(stmt->ext.proc_param, ""); + return stmt->ext.proc_param(stmt); +} + +static SQLRETURN do_process_param_row(stmt_t *stmt) { + paramset_t *paramset = &stmt->paramset; + int n_params = paramset->n_params; + + SQLRETURN r = SQL_SUCCESS; + + paramset->i_col = 0; + for (; paramset->i_coli_col) { + r = do_process_param(stmt); + if (r!=SQL_SUCCESS) return r; + } + + return SQL_SUCCESS; +} + +stmt_t* stmt_new(conn_t *conn) { + stmt_t *stmt = (stmt_t*)calloc(1, sizeof(*stmt)); + if (!stmt) return NULL; + + if (stmt_init(stmt, conn)) { + free(stmt); + return NULL; + } + + return stmt; +} + +void stmt_free(stmt_t *stmt) { + if (!stmt) return; + + // clean ext stuff + if (stmt->ext.free_stmt) { + stmt->ext.free_stmt(stmt); + stmt->ext.free_stmt = NULL; + } + + // clean stmt stuff + stmt_release(stmt); + + free(stmt); +} + +conn_t* stmt_get_conn(stmt_t *stmt) { + if (!stmt) return NULL; + if (!stmt->owner) return NULL; + + return stmt->owner->conn; +} + +void stmt_reclaim_params(stmt_t *stmt) { + if (!stmt) return; + paramset_reclaim_params(&stmt->paramset); +} + +void stmt_reclaim_param_bindings(stmt_t *stmt) { + if (!stmt) return; + paramset_reclaim_bindings(&stmt->paramset); +} + +void stmt_reclaim_field_bindings(stmt_t *stmt) { + if (!stmt) return; + fieldset_reclaim_bindings(&stmt->fieldset); +} + +void stmt_close_rs(stmt_t *stmt) { + if (!stmt) return; + + OILE(stmt->ext.close_rs, ""); + stmt->ext.close_rs(stmt); + + stmt->typeinfo = SQL_UNKNOWN_TYPE; + + stmt->eof = 0; + + fieldset_reclaim_fields(&stmt->fieldset); + // for the performance + // we don't reclaim field-binds here + // https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlbindcol-function?view=sql-server-ver15 + STMT_SET_NORM(stmt); +} + +void stmt_reset_params(stmt_t *stmt) { + if (!stmt) return; + + stmt->attr.paramset_size = 0; + stmt_reclaim_param_bindings(stmt); +} + +static SQLRETURN stmt_set_sql(stmt_t *stmt, todbc_string_t *txt) { + OILE(stmt, ""); + OILE(txt, ""); + errs_t *errs = &stmt->errs; + sql_t *sql = &stmt->sql; + + OILE(txt!=&sql->txt, ""); + + todbc_buf_t *cache = sql->cache; + OILE(sql->cache, ""); + + conn_t *conn = stmt_get_conn(stmt); + OILE(conn, ""); + const char *enc = conn->enc_db; + + todbc_buf_reclaim(cache); + + sql->txt = todbc_string_conv_to(txt, enc, cache); + if (!sql->txt.buf) { + SET_OOM(errs, ""); + return SQL_ERROR; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_stmt_prepare(stmt_t *stmt) { + OILE(stmt->ext.prepare, ""); + return stmt->ext.prepare(stmt); +} + +static void do_stmt_clear_param_vals(stmt_t *stmt) { + OILE(stmt->ext.clear_param_vals, ""); + stmt->ext.clear_param_vals(stmt); +} + +static void do_stmt_clear_params(stmt_t *stmt) { + stmt->prepared = 0; + paramset_t *paramset = &stmt->paramset; + paramset_reclaim_params(paramset); + do_stmt_clear_param_vals(stmt); +} + +SQLRETURN stmt_exec_direct(stmt_t *stmt, todbc_string_t *txt) { + OILE(stmt, ""); + OILE(txt, ""); + + SQLRETURN r = stmt_set_sql(stmt, txt); + if (r!=SQL_SUCCESS) return r; + + do_stmt_clear_params(stmt); + + if (stmt->ext.exec_direct) { + r = stmt->ext.exec_direct(stmt); + } else { + r = do_stmt_prepare(stmt); + if (r!=SQL_SUCCESS) return r; + stmt->prepared = 1; + OILE(0, ""); + } + + return r; +} + +SQLRETURN stmt_prepare(stmt_t *stmt, todbc_string_t *txt) { + OILE(stmt, ""); + OILE(txt, ""); + + SQLRETURN r = stmt_set_sql(stmt, txt); + if (r!=SQL_SUCCESS) return r; + + do_stmt_clear_params(stmt); + + r = do_stmt_prepare(stmt); + if (r!=SQL_SUCCESS) return r; + stmt->prepared = 1; + + return SQL_SUCCESS; +} + +SQLRETURN stmt_bind_param(stmt_t *stmt, param_binding_t *arg) { + OILE(stmt, ""); + OILE(arg, ""); + OILE(arg->ParameterNumber>0, ""); + int idx = arg->ParameterNumber - 1; + + errs_t *errs = &stmt->errs; + paramset_t *paramset = &stmt->paramset; + + paramset_init_bindings_cache(paramset); + if (!paramset->bindings_cache) { + SET_OOM(errs, "failed to alloc cache for param binds"); + return SQL_ERROR; + } + + todbc_buf_t *cache = paramset->bindings_cache; + OILE(cache, ""); + + param_binding_t *bindings = paramset->bindings; + if (idx>=paramset->n_bindings) { + size_t num = (size_t)(idx + 1); + // align + const size_t block = 10; + num = (num + block-1) / block * block; + bindings = (param_binding_t*)todbc_buf_realloc(cache, bindings, num * sizeof(*bindings)); + if (!bindings) { + SET_OOM(errs, "failed to realloc buf for param binds"); + return SQL_ERROR; + } + paramset->bindings = bindings; + paramset->n_bindings = idx + 1; + } + OILE(paramset->bindings, ""); + OILE(idxn_bindings, ""); + + param_binding_t *binding = bindings + idx; + *binding = *arg; + + if (paramset->n_bindings>paramset->n_params) return SQL_SUCCESS; + + OILE(stmt->ext.set_param_conv, ""); + return stmt->ext.set_param_conv(stmt, idx); +} + +SQLRETURN stmt_execute(stmt_t *stmt) { + OILE(stmt, ""); + OILE(!STMT_IS_EXECUTING(stmt), ""); + + SQLRETURN r = SQL_SUCCESS; + + if (stmt->prepared==0) { + do_stmt_clear_params(stmt); + + r = do_stmt_prepare(stmt); + if (r!=SQL_SUCCESS) return r; + stmt->prepared = 1; + } + OILE(stmt->prepared==1, ""); + + + errs_t *errs = &stmt->errs; + + paramset_t *paramset = &stmt->paramset; + int n_params = paramset->n_params; + int n_bindings = paramset->n_bindings; + + if (n_params>n_bindings) { + SET_GENERAL(errs, "parameters need to be bound first"); + return SQL_ERROR; + } + + if (n_params>0) { + int paramset_size = (int)stmt->attr.paramset_size; + if (paramset_size==0) paramset_size = 1; + stmt->attr.paramset_size = (SQLULEN)paramset_size; + + paramset->i_row = 0; + for (; paramset->i_rowi_row) { + r = do_process_param_row(stmt); + if (r) return r; + + OILE(stmt->ext.param_row_processed, ""); + r = stmt->ext.param_row_processed(stmt); + if (r) return r; + } + } + + OILE(stmt->ext.execute, ""); + return stmt->ext.execute(stmt); +} + +SQLRETURN stmt_fetch(stmt_t *stmt) { + OILE(stmt, ""); + OILE(STMT_IS_EXECUTED(stmt), ""); + + if (stmt->eof) return SQL_NO_DATA; + + OILE(stmt->ext.fetch, ""); + SQLRETURN r = stmt->ext.fetch(stmt); + if (r!=SQL_SUCCESS) return r; + + fieldset_t *fieldset = &stmt->fieldset; + if (fieldset->n_bindings==0) return SQL_SUCCESS; + OILE(fieldset->n_bindings>0, ""); + + for (size_t i=0; in_bindings; ++i) { + OILE(fieldset->bindings, ""); + col_binding_t *binding = fieldset->bindings + i; + if (binding->ColumnNumber!=i+1) { + OILE(binding->ColumnNumber==0, ""); + continue; + } + OILE(stmt->ext.get_data, ""); + r = stmt->ext.get_data(stmt, binding); + if (r!=SQL_SUCCESS) return r; + } + + return SQL_SUCCESS; +} + +SQLRETURN stmt_get_data(stmt_t *stmt, col_binding_t *binding) { + OILE(stmt, ""); + OILE(STMT_IS_EXECUTED(stmt), ""); + + OILE(stmt->eof==0, ""); + + OILE(stmt->ext.get_data, ""); + return stmt->ext.get_data(stmt, binding); +} + +SQLRETURN stmt_bind_col(stmt_t *stmt, col_binding_t *binding) { + OILE(stmt, ""); + // shall we check execute state? + + errs_t *errs = &stmt->errs; + + fieldset_t *fieldset = &stmt->fieldset; + + todbc_buf_t *cache = fieldset->bindings_cache; + if (cache==NULL) { + fieldset_init_bindings(fieldset); + cache = fieldset->bindings_cache; + if (!cache) { + SET_OOM(errs, ""); + return SQL_ERROR; + } + } + OILE(cache, ""); + + col_binding_t *bindings = fieldset->bindings; + + OILE(binding->ColumnNumber>0, ""); + if (binding->ColumnNumber>=fieldset->n_bindings) { + size_t num = (size_t)binding->ColumnNumber; + const size_t block = 10; + size_t align = (num+block-1)/block*block; + size_t total = align * sizeof(*bindings); + bindings = (col_binding_t*)todbc_buf_realloc(cache, bindings, total); + if (!bindings) { + SET_OOM(errs, ""); + return SQL_ERROR; + } + for (size_t i = (size_t)fieldset->n_bindings; ibindings = bindings; + fieldset->n_bindings = (int)num; + } + OILE(bindings, ""); + OILE(binding->ColumnNumber<=fieldset->n_bindings, ""); + bindings[binding->ColumnNumber-1] = *binding; + + return SQL_SUCCESS; +} + + + + +// public +errs_t* stmt_get_errs(stmt_t *stmt) { + OILE(stmt, ""); + + return &stmt->errs; +} + +void stmt_clr_errs(stmt_t *stmt) { + if (!stmt) return; + + errs_reclaim(&stmt->errs); +} + diff --git a/src/connector/odbc/src/base/stmt.h b/src/connector/odbc/src/base/stmt.h new file mode 100644 index 0000000000000000000000000000000000000000..905cb6a00323446736f130642d0e250e4ba24dd9 --- /dev/null +++ b/src/connector/odbc/src/base/stmt.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _stmt_h_ +#define _stmt_h_ + +#include "../base.h" + +#include "err.h" +#include "field.h" +#include "param.h" +#include "rs.h" + +typedef struct descs_s descs_t; +typedef struct stmts_s stmts_t; +typedef struct stmt_ext_s stmt_ext_t; +typedef struct stmt_attr_s stmt_attr_t; +typedef struct sql_s sql_t; + +struct descs_s { + void *app_row; // SQL_ATTR_APP_ROW_DESC + void *app_param; // SQL_ATTR_APP_PARAM_DESC + void *imp_row; // SQL_ATTR_IMP_ROW_DESC + void *imp_param; // SQL_ATTR_IMP_PARAM_DESC +}; + +struct stmt_ext_s { + void *ext; + void (*free_stmt)(stmt_t *stmt); + + void (*clear_params)(stmt_t *stmt); + void (*clear_param_vals)(stmt_t *stmt); + + SQLRETURN (*get_param)(stmt_t *stmt, param_t *arg); + SQLRETURN (*set_param_conv)(stmt_t *stmt, int idx); + + SQLRETURN (*exec_direct)(stmt_t *stmt); + SQLRETURN (*prepare)(stmt_t *stmt); + SQLRETURN (*proc_param)(stmt_t *stmt); + SQLRETURN (*param_row_processed)(stmt_t *stmt); + SQLRETURN (*execute)(stmt_t *stmt); + SQLRETURN (*get_affected_rows)(stmt_t *stmt, SQLLEN *RowCount); + SQLRETURN (*get_fields_count)(stmt_t *stmt, SQLSMALLINT *ColumnCount); + SQLRETURN (*get_field)(stmt_t *stmt, field_arg_t *arg); + SQLRETURN (*fetch)(stmt_t *stmt); + SQLRETURN (*get_data)(stmt_t *stmt, col_binding_t *col); + void (*close_rs)(stmt_t *stmt); +}; + +struct stmt_attr_s { + // SQL_ATTR_PARAM_BIND_TYPE: SQL_PARAM_BIND_BY_COLUMN or row size + SQLULEN bind_type; // default: SQL_PARAM_BIND_BY_COLUMN + // SQL_ATTR_PARAM_BIND_OFFSET_PTR + SQLULEN *bind_offset_ptr; // default: NULL + // SQL_ATTR_PARAMSET_SIZE + SQLULEN paramset_size; // default: 0 +}; + +struct sql_s { + todbc_buf_t *cache; + todbc_string_t txt; +}; + +struct stmt_s { + stmts_t *owner; + stmt_t *next; + stmt_t *prev; + + SQLSMALLINT typeinfo; + + descs_t descs; + + sql_t sql; + + stmt_attr_t attr; + + paramset_t paramset; + fieldset_t fieldset; + + int affected_rows; + rs_t rs; + + errs_t errs; + + stmt_ext_t ext; + + unsigned int prepared:1; + unsigned int execute:2; + unsigned int eof:1; +}; + +struct stmts_s { + conn_t *conn; + + int count; + stmt_t *head; + stmt_t *tail; +}; + +#define STMT_TYPEINFO(stmt) (stmt->typeinfo!=SQL_UNKNOWN_TYPE) + +#define STMT_SET_EXECUTING(stmt) (stmt->execute=0x01) +#define STMT_SET_EXECUTED(stmt) (stmt->execute=0x02) +#define STMT_SET_NORM(stmt) (stmt->execute=0x00) + +#define STMT_IS_EXECUTING(stmt) (stmt->execute==0x01) +#define STMT_IS_EXECUTED(stmt) (stmt->execute==0x02) +#define STMT_IS_NORM(stmt) (stmt->execute==0x00) + +stmt_t* stmt_new(conn_t *conn); +void stmt_free(stmt_t *stmt); + +conn_t* stmt_get_conn(stmt_t *stmt); + +void stmt_reclaim_params(stmt_t *stmt); +void stmt_reclaim_param_binds(stmt_t *stmt); +void stmt_reclaim_field_binds(stmt_t *stmt); +void stmt_close_rs(stmt_t *stmt); +void stmt_reset_params(stmt_t *stmt); +SQLRETURN stmt_exec_direct(stmt_t *stmt, todbc_string_t *txt); +SQLRETURN stmt_prepare(stmt_t *stmt, todbc_string_t *txt); +SQLRETURN stmt_bind_param(stmt_t *stmt, param_binding_t *arg); +SQLRETURN stmt_execute(stmt_t *stmt); +SQLRETURN stmt_fetch(stmt_t *stmt); +SQLRETURN stmt_get_data(stmt_t *stmt, col_binding_t *binding); +SQLRETURN stmt_bind_col(stmt_t *stmt, col_binding_t *binding); + +#endif // _stmt_h_ + + + diff --git a/src/connector/odbc/src/base/tsdb_impl.c b/src/connector/odbc/src/base/tsdb_impl.c new file mode 100644 index 0000000000000000000000000000000000000000..d913aebd7df55ceef337d48131f7bab634219e1b --- /dev/null +++ b/src/connector/odbc/src/base/tsdb_impl.c @@ -0,0 +1,2598 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb_impl.h" + +#include "env.h" +#include "../todbc_tls.h" +#include "../todbc_util.h" + +#include "ttype.h" + + +#include + +// tsdb_conn + +typedef struct tsdb_conn_s tsdb_conn_t; +typedef struct tsdb_stmt_s tsdb_stmt_t; +typedef struct tsdb_param_s tsdb_param_t; +typedef struct tsdb_param_val_s tsdb_param_val_t; +typedef struct tsdb_param_conv_arg_s tsdb_param_conv_arg_t; + +struct tsdb_conn_s { + conn_t *conn; + + char svr_info[64]; + char cli_info[64]; + + TAOS *taos; +}; + +struct tsdb_param_s { + int tsdb_type; // TSDB_DATA_TYPE_xxx + int tsdb_bytes; + + SQLRETURN (*conv)(stmt_t *stmt, tsdb_param_conv_arg_t *arg); +}; + +struct tsdb_param_val_s { + SQLUSMALLINT ParameterNumber; + int is_null; +}; + +struct tsdb_stmt_s { + stmt_t *stmt; + + TAOS_STMT *tsdb_stmt; // prepared-statement + TAOS_RES *tsdb_res; + + tsdb_param_t *tsdb_params; + + todbc_buf_t *tsdb_param_vals_cache; + tsdb_param_val_t *tsdb_param_vals; + TAOS_BIND *taos_binds; + + TAOS_FIELD *tsdb_fields; + TAOS_ROW tsdb_curr; + + unsigned int by_query:1; +}; + +struct tsdb_param_conv_arg_s { + conn_t *conn; + todbc_buf_t *cache; + int idx; + SQLPOINTER val; + SQLLEN soi; + tsdb_param_t *tsdb_param; + tsdb_param_val_t *tsdb_param_val; + TAOS_BIND *taos_bind; +}; + +static void tsdb_stmt_init_param_vals_cache(tsdb_stmt_t *tsdb_stmt) { + OILE(tsdb_stmt, ""); + if (tsdb_stmt->tsdb_param_vals_cache) return; + tsdb_stmt->tsdb_param_vals_cache = todbc_buf_create(); +} + +static void tsdb_stmt_reclaim_param_vals(tsdb_stmt_t *tsdb_stmt) { + if (!tsdb_stmt) return; + if (tsdb_stmt->tsdb_param_vals_cache) { + tsdb_stmt->tsdb_param_vals = NULL; + tsdb_stmt->taos_binds = NULL; + todbc_buf_reclaim(tsdb_stmt->tsdb_param_vals_cache); + } + OILE(tsdb_stmt->tsdb_param_vals==NULL, ""); + OILE(tsdb_stmt->taos_binds==NULL, ""); +} + +static void tsdb_stmt_cleanup_param_vals(tsdb_stmt_t *tsdb_stmt) { + if (!tsdb_stmt) return; + tsdb_stmt_reclaim_param_vals(tsdb_stmt); + + if (tsdb_stmt->tsdb_param_vals_cache) { + todbc_buf_free(tsdb_stmt->tsdb_param_vals_cache); + } +} + +static void tsdb_stmt_calloc_param_vals(tsdb_stmt_t *tsdb_stmt) { + OILE(tsdb_stmt, ""); + stmt_t *stmt = tsdb_stmt->stmt; + OILE(stmt, ""); + paramset_t *paramset = &stmt->paramset; + int n_params = paramset->n_params; + OILE(n_params>0, ""); + todbc_buf_t *cache = tsdb_stmt->tsdb_param_vals_cache; + OILE(cache, ""); + OILE(tsdb_stmt->tsdb_param_vals==NULL, ""); + OILE(tsdb_stmt->taos_binds==NULL, ""); + tsdb_stmt->tsdb_param_vals = (tsdb_param_val_t*)todbc_buf_calloc(cache, (size_t)n_params, sizeof(*tsdb_stmt->tsdb_param_vals)); + tsdb_stmt->taos_binds = (TAOS_BIND*)todbc_buf_calloc(cache, (size_t)n_params, sizeof(*tsdb_stmt->taos_binds)); +} + +static SQLRETURN tsdb_stmt_init_stmt(tsdb_stmt_t *tsdb_stmt) { + OILE(tsdb_stmt && tsdb_stmt->stmt, ""); + errs_t *errs = &tsdb_stmt->stmt->errs; + + if (tsdb_stmt->tsdb_stmt) return SQL_SUCCESS; + OILE(tsdb_stmt->stmt->owner, ""); + conn_t *conn = tsdb_stmt->stmt->owner->conn; + OILE(conn, ""); + tsdb_conn_t *tsdb_conn = (tsdb_conn_t*)conn->ext.ext; + OILE(tsdb_conn && tsdb_conn->taos, ""); + tsdb_stmt->tsdb_stmt = taos_stmt_init(tsdb_conn->taos); + if (!tsdb_stmt->tsdb_stmt) { + SET_GENERAL(errs, "failed to init taos stmt"); + return SQL_ERROR; + } + + return SQL_SUCCESS; +} + +static void tsdb_stmt_close_stmt(tsdb_stmt_t *tsdb_stmt) { + if (!tsdb_stmt) return; + if (!tsdb_stmt->tsdb_stmt) return; + + int r = taos_stmt_close(tsdb_stmt->tsdb_stmt); + tsdb_stmt->tsdb_stmt= NULL; + tsdb_stmt->stmt->prepared = 0; + if (r) OD("[%d]%s", r, tstrerror(r)); +} + +static void tsdb_stmt_close_rs(tsdb_stmt_t *tsdb_stmt) { + if (!tsdb_stmt) return; + if (!tsdb_stmt->tsdb_res) return; + + tsdb_stmt->tsdb_curr = NULL; + + if (tsdb_stmt->by_query) { + taos_stop_query(tsdb_stmt->tsdb_res); + } else { + OILE(tsdb_stmt->tsdb_stmt==NULL, ""); + taos_free_result(tsdb_stmt->tsdb_res); + } + tsdb_stmt->tsdb_res = NULL; +} + +static void tsdb_conn_free(conn_t *conn) { + OILE(conn, ""); + + tsdb_conn_t *tsdb_conn = (tsdb_conn_t*)conn->ext.ext; + OILE(tsdb_conn, ""); + OILE(tsdb_conn->taos==NULL, ""); + + conn->ext.ext = NULL; + conn->ext.free_conn = NULL; + + free(tsdb_conn); +} + +static SQLRETURN tsdb_conn_connect(conn_t *conn) { + OILE(conn, ""); + OILE(CONN_IS_NORM(conn), ""); + errs_t *errs = &conn->errs; + tsdb_conn_t *tsdb_conn = (tsdb_conn_t*)conn->ext.ext; + OILE(tsdb_conn, ""); + OILE(tsdb_conn->conn==conn, ""); + + conn_val_t *val = &conn->val; + const char *dsn = val->dsn; + const char *uid = val->uid; + const char *pwd = val->pwd; + const char *db = val->db; + const char *svr = val->server; + + OILE(dsn, ""); + + int use_default = 0; + char server[4096]; server[0] = '\0'; + if (!svr || !svr[0]) { + int n = SQLGetPrivateProfileString(dsn, "Server", "", server, sizeof(server)-1, "Odbc.ini"); + if (n<=0) { + snprintf(server, sizeof(server), DEFAULT_SERVER); + n = (int)strlen(server); + use_default = 1; + } else { + server[n] = '\0'; + } + svr = server; + + if (!svr || !svr[0]) { + SET_GENERAL(errs, "please specify Server entry in connection string or odbc.ini or windows registry for DSN:[%s]", dsn); + return SQL_ERROR; + } + } + + char *ip = NULL; + int port = 0; + char *p = strchr(svr, ':'); + if (p) { + ip = todbc_tls_strndup(svr, (size_t)(p-svr)); + port = atoi(p+1); + } + + tsdb_conn->taos = taos_connect(ip, uid, pwd, db, (uint16_t)port); + if (!tsdb_conn->taos) { + int e = terrno; + const char * es = tstrerror(e); + if (use_default) { + SET_GENERAL(errs, "no Server entry in odbc.ini or windows registry for DSN[%s], fallback to svr[%s:%d] db[%s]", dsn, ip, port, db); + } + SET_GENERAL(errs, "connect to DSN[%s] svr[%s:%d] db[%s] failed", dsn, ip, port, db); + SET_GENERAL(errs, "[%x]%s", e, es); + return SQL_ERROR; + } + + const char *svr_info = taos_get_server_info(tsdb_conn->taos); + const char *cli_info = taos_get_client_info(tsdb_conn->taos); + snprintf(tsdb_conn->svr_info, sizeof(tsdb_conn->svr_info), "%s", svr_info); + snprintf(tsdb_conn->cli_info, sizeof(tsdb_conn->cli_info), "%s", cli_info); + + return SQL_SUCCESS; +} + +static void tsdb_conn_disconnect(conn_t *conn) { + OILE(conn, ""); + OILE(CONN_IS_CONNECTED(conn), ""); + + tsdb_conn_t *tsdb_conn = (tsdb_conn_t*)conn->ext.ext; + OILE(tsdb_conn, ""); + OILE(tsdb_conn->conn==conn, ""); + + TAOS *taos = tsdb_conn->taos; + + if (!taos) return; + taos_close(taos); + taos = NULL; + tsdb_conn->taos = NULL; +} + +static void tsdb_conn_free_stmt(stmt_t *stmt) { + OILE(stmt, ""); + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt && tsdb_stmt->stmt==stmt, ""); + + tsdb_stmt_close_rs(tsdb_stmt); + tsdb_stmt_close_stmt(tsdb_stmt); + tsdb_stmt_cleanup_param_vals(tsdb_stmt); + + tsdb_stmt->tsdb_params = NULL; + + stmt_ext_t ext = {0}; + stmt->ext = ext; + + free(tsdb_stmt); +} + +static void tsdb_conn_clear_param_vals(stmt_t *stmt) { + OILE(stmt, ""); + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt && tsdb_stmt->stmt==stmt, ""); + + tsdb_stmt_reclaim_param_vals(tsdb_stmt); +} + +static SQLRETURN tsdb_conn_exec_direct(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + OILE(!STMT_IS_EXECUTED(stmt), ""); + + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + tsdb_conn_t *tsdb_conn = (tsdb_conn_t*)conn->ext.ext; + OILE(tsdb_conn && tsdb_conn->conn==conn, ""); + TAOS *taos = tsdb_conn->taos; + OILE(taos, ""); + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt && tsdb_stmt->stmt==stmt, ""); + OILE(tsdb_stmt->tsdb_stmt==NULL, ""); + OILE(tsdb_stmt->tsdb_res==NULL, ""); + + errs_t *errs = &stmt->errs; + + tsdb_stmt_reclaim_param_vals(tsdb_stmt); + + const char *txt = (const char*)stmt->sql.txt.buf; + TAOS_RES *tsdb_res = taos_query(taos, txt); + OILE(tsdb_res, ""); + int r = taos_errno(tsdb_res); + if (r) { + SET_GENERAL(errs, "taos query failed:[%d]%s", r, tstrerror(r)); + taos_stop_query(tsdb_res); + STMT_SET_NORM(stmt); + return SQL_ERROR; + } + + tsdb_stmt->tsdb_res = tsdb_res; + tsdb_stmt->by_query = 1; + + STMT_SET_EXECUTED(stmt); + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_int64_to_tsdb_timestamp(stmt_t *stmt, const int64_t v, tsdb_param_conv_arg_t *arg) { + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + taos_bind->u.ts = v; + taos_bind->buffer_length = sizeof(taos_bind->u.ts); + taos_bind->buffer = &taos_bind->u.ts; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_int64_to_tsdb_tinyint(stmt_t *stmt, const int64_t v, tsdb_param_conv_arg_t *arg) { + errs_t *errs = &stmt->errs; + + if (vINT8_MAX) { + SET_GENERAL(errs, "integer overflow for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_TINYINT; + taos_bind->u.v1 = (int8_t)v; + taos_bind->buffer_length = sizeof(taos_bind->u.v1); + taos_bind->buffer = &taos_bind->u.v1; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_int64_to_tsdb_smallint(stmt_t *stmt, const int64_t v, tsdb_param_conv_arg_t *arg) { + errs_t *errs = &stmt->errs; + + if (vINT16_MAX) { + SET_GENERAL(errs, "integer overflow for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; + taos_bind->u.v2 = (int16_t)v; + taos_bind->buffer_length = sizeof(taos_bind->u.v2); + taos_bind->buffer = &taos_bind->u.v2; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_int64_to_tsdb_int(stmt_t *stmt, const int64_t v, tsdb_param_conv_arg_t *arg) { + errs_t *errs = &stmt->errs; + + if (vINT32_MAX) { + SET_GENERAL(errs, "integer overflow for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_INT; + taos_bind->u.v4 = (int32_t)v; + taos_bind->buffer_length = sizeof(taos_bind->u.v4); + taos_bind->buffer = &taos_bind->u.v4; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_int64_to_tsdb_bigint(stmt_t *stmt, const int64_t v, tsdb_param_conv_arg_t *arg) { + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_BIGINT; + taos_bind->u.v8 = v; + taos_bind->buffer_length = sizeof(taos_bind->u.v8); + taos_bind->buffer = &taos_bind->u.v8; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_double_to_tsdb_float(stmt_t *stmt, const double v, tsdb_param_conv_arg_t *arg) { + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_FLOAT; + taos_bind->u.f4 = (float)v; + taos_bind->buffer_length = sizeof(taos_bind->u.f4); + taos_bind->buffer = &taos_bind->u.f4; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_double_to_tsdb_double(stmt_t *stmt, const double v, tsdb_param_conv_arg_t *arg) { + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; + taos_bind->u.f8 = v; + taos_bind->buffer_length = sizeof(taos_bind->u.f8); + taos_bind->buffer = &taos_bind->u.f8; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + + +static SQLRETURN do_conv_sql_string_to_int64(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg, int64_t *v) { + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt.buf; + int bytes = 0; + int64_t i64 = 0; + sscanf(buf, "%" PRId64 " %n", &i64, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to integer for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + *v = i64; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_double(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg, double *v) { + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt.buf; + int bytes = 0; + double dbl = 0.0; + sscanf(buf, "%lf%n", &dbl, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to double for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + *v = dbl; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_timestamp(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + char *buf = (char*)txt.buf; + int32_t bytes = (int32_t)txt.bytes; + + int64_t ts = 0; + int r = taosParseTime(buf, &ts, bytes, TSDB_TIME_PRECISION_MILLI, 0); + if (r) { + SET_GENERAL(errs, "failed to parse as timestamp for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + taos_bind->u.ts = ts; + taos_bind->buffer_length = sizeof(taos_bind->u.ts); + taos_bind->buffer = &taos_bind->u.ts; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_nchar(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_db; + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt_db = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt_db.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt_db.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + char *buf = (char*)txt_db.buf; + int32_t bytes = (int32_t)txt_db.bytes; + if (bytes > arg->tsdb_param->tsdb_bytes) { + SET_OOM(errs, "failed to convert from [%s->%s] for param [%d], string too long [%d/%d]", + enc_from, enc_to, arg->idx+1, bytes, arg->tsdb_param->tsdb_bytes); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_NCHAR; + taos_bind->u.nchar = buf; + taos_bind->buffer_length = (uintptr_t)((size_t)bytes); + taos_bind->buffer = taos_bind->u.nchar; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_bool(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + int64_t v = 0; + + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt_db = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt_db.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt_db.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt_db.buf; + int bytes = 0; + if (strcasecmp(buf, "true")==0) { + v = 1; + } else if (strcasecmp(buf, "false")==0) { + v = 0; + } else { + sscanf(buf, "%" PRId64 " %n", &v, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to integer for param [%d]", arg->idx+1); + return SQL_ERROR; + } + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_BOOL; + taos_bind->u.b = v ? 1 : 0; + taos_bind->buffer_length = sizeof(taos_bind->u.b); + taos_bind->buffer = &taos_bind->u.b; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_tinyint(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + int64_t v = 0; + + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt_db = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt_db.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt_db.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt_db.buf; + int bytes = 0; + sscanf(buf, "%" PRId64 " %n", &v, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to integer for param [%d]", arg->idx+1); + return SQL_ERROR; + } + if (vINT8_MAX) { + SET_GENERAL(errs, "failed to convert to tinyint for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_TINYINT; + taos_bind->u.v1 = (int8_t)v; + taos_bind->buffer_length = sizeof(taos_bind->u.v1); + taos_bind->buffer = &taos_bind->u.v1; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_smallint(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + int64_t v = 0; + + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt_db = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt_db.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt_db.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt_db.buf; + int bytes = 0; + sscanf(buf, "%" PRId64 " %n", &v, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to integer for param [%d]", arg->idx+1); + return SQL_ERROR; + } + if (vINT16_MAX) { + SET_GENERAL(errs, "failed to convert to smallint for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_SMALLINT; + taos_bind->u.v2 = (int16_t)v; + taos_bind->buffer_length = sizeof(taos_bind->u.v2); + taos_bind->buffer = &taos_bind->u.v2; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_int(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + int64_t v = 0; + + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt_db = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt_db.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt_db.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt_db.buf; + int bytes = 0; + sscanf(buf, "%" PRId64 " %n", &v, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to integer for param [%d]", arg->idx+1); + return SQL_ERROR; + } + if (vINT32_MAX) { + SET_GENERAL(errs, "failed to convert to int for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_INT; + taos_bind->u.v4 = (int32_t)v; + taos_bind->buffer_length = sizeof(taos_bind->u.v4); + taos_bind->buffer = &taos_bind->u.v4; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_bigint(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + int64_t v = 0; + + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt_db = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt_db.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt_db.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt_db.buf; + int bytes = 0; + sscanf(buf, "%" PRId64 " %n", &v, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to integer for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_BIGINT; + taos_bind->u.v8 = (int64_t)v; + taos_bind->buffer_length = sizeof(taos_bind->u.v8); + taos_bind->buffer = &taos_bind->u.v8; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_float(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + double v = 0; + + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt_db = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt_db.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt_db.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt_db.buf; + int bytes = 0; + sscanf(buf, "%lf %n", &v, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to integer for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_FLOAT; + taos_bind->u.f4 = (float)v; + taos_bind->buffer_length = sizeof(taos_bind->u.f4); + taos_bind->buffer = &taos_bind->u.f4; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_string_to_tsdb_double(stmt_t *stmt, const char *enc_from, tsdb_param_conv_arg_t *arg) { + double v = 0; + + todbc_buf_t *cache = arg->cache; + const char *enc_to = arg->conn->enc_src; // ?locale or src?, windows iconv!!! + const unsigned char *src = (const unsigned char*)arg->val; + size_t slen = (size_t)arg->soi; + + errs_t *errs = &stmt->errs; + + todbc_string_t txt_db = todbc_tls_conv(cache, enc_to, enc_from, src, &slen); + if (!txt_db.buf) { + SET_OOM(errs, "failed to alloc space to convert from [%s->%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + if (txt_db.bytes%s] for param [%d]", enc_from, enc_to, arg->idx+1); + return SQL_ERROR; + } + + const char *buf = (const char*)txt_db.buf; + int bytes = 0; + sscanf(buf, "%lf %n", &v, &bytes); + if (strlen(buf)!=bytes) { + SET_GENERAL(errs, "failed to convert to integer for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_DOUBLE; + taos_bind->u.f8 = v; + taos_bind->buffer_length = sizeof(taos_bind->u.f8); + taos_bind->buffer = &taos_bind->u.f8; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_timestamp(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + return do_conv_sql_string_to_tsdb_timestamp(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_nchar(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + return do_conv_sql_string_to_tsdb_nchar(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_bool(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + return do_conv_sql_string_to_tsdb_bool(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_tinyint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + int64_t v = 0; + SQLRETURN r = do_conv_sql_string_to_int64(stmt, enc_from, arg, &v); + if (r!=SQL_SUCCESS) return r; + return do_conv_int64_to_tsdb_tinyint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_smallint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + int64_t v = 0; + SQLRETURN r = do_conv_sql_string_to_int64(stmt, enc_from, arg, &v); + if (r!=SQL_SUCCESS) return r; + return do_conv_int64_to_tsdb_smallint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_int(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + int64_t v = 0; + SQLRETURN r = do_conv_sql_string_to_int64(stmt, enc_from, arg, &v); + if (r!=SQL_SUCCESS) return r; + return do_conv_int64_to_tsdb_int(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_bigint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + int64_t v = 0; + SQLRETURN r = do_conv_sql_string_to_int64(stmt, enc_from, arg, &v); + if (r!=SQL_SUCCESS) return r; + return do_conv_int64_to_tsdb_bigint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_float(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + double v = 0; + SQLRETURN r = do_conv_sql_string_to_double(stmt, enc_from, arg, &v); + if (r!=SQL_SUCCESS) return r; + return do_conv_double_to_tsdb_float(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_wchar_to_tsdb_double(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_wchar; + double v = 0; + SQLRETURN r = do_conv_sql_string_to_double(stmt, enc_from, arg, &v); + if (r!=SQL_SUCCESS) return r; + return do_conv_double_to_tsdb_double(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_timestamp(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_timestamp(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_nchar(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_nchar(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_bool(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_bool(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_tinyint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_tinyint(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_smallint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_smallint(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_int(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_int(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_bigint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_bigint(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_float(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_float(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_sql_char_to_tsdb_double(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + const char *enc_from = arg->conn->enc_char; + return do_conv_sql_string_to_tsdb_double(stmt, enc_from, arg); +} + +static SQLRETURN do_conv_int64_to_tsdb_bool(stmt_t *stmt, const int64_t v, tsdb_param_conv_arg_t *arg) { + errs_t *errs = &stmt->errs; + + if (v!=1 && v!=0) { + SET_GENERAL(errs, "integer overflow for param [%d]", arg->idx+1); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_BOOL; + taos_bind->u.b = v ? 1 : 0; + taos_bind->buffer_length = sizeof(taos_bind->u.b); + taos_bind->buffer = &taos_bind->u.b; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_sql_sbigint_to_tsdb_bigint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int64_t v = *(int64_t*)arg->val; + + return do_conv_int64_to_tsdb_bigint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_sbigint_to_tsdb_tinyint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int64_t v = *(int64_t*)arg->val; + + return do_conv_int64_to_tsdb_tinyint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_sbigint_to_tsdb_int(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int64_t v = *(int64_t*)arg->val; + + return do_conv_int64_to_tsdb_int(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_sbigint_to_tsdb_timestamp(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int64_t v = *(int64_t*)arg->val; + + return do_conv_int64_to_tsdb_timestamp(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_long_to_tsdb_bool(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int32_t v = *(int32_t*)arg->val; + + return do_conv_int64_to_tsdb_bool(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_long_to_tsdb_tinyint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int32_t v = *(int32_t*)arg->val; + + return do_conv_int64_to_tsdb_tinyint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_long_to_tsdb_smallint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int32_t v = *(int32_t*)arg->val; + + return do_conv_int64_to_tsdb_smallint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_long_to_tsdb_int(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int32_t v = *(int32_t*)arg->val; + + return do_conv_int64_to_tsdb_int(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_long_to_tsdb_bigint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int32_t v = *(int32_t*)arg->val; + + return do_conv_int64_to_tsdb_bigint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_tinyint_to_tsdb_bool(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int8_t v = *(int8_t*)arg->val; + + return do_conv_int64_to_tsdb_bool(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_tinyint_to_tsdb_tinyint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int8_t v = *(int8_t*)arg->val; + + return do_conv_int64_to_tsdb_tinyint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_tinyint_to_tsdb_smallint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int8_t v = *(int8_t*)arg->val; + + return do_conv_int64_to_tsdb_smallint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_tinyint_to_tsdb_int(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int8_t v = *(int8_t*)arg->val; + + return do_conv_int64_to_tsdb_int(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_tinyint_to_tsdb_bigint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int8_t v = *(int8_t*)arg->val; + + return do_conv_int64_to_tsdb_bigint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_short_to_tsdb_bool(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int16_t v = *(int16_t*)arg->val; + + return do_conv_int64_to_tsdb_bool(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_short_to_tsdb_tinyint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int16_t v = *(int16_t*)arg->val; + + return do_conv_int64_to_tsdb_tinyint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_short_to_tsdb_smallint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int16_t v = *(int16_t*)arg->val; + + return do_conv_int64_to_tsdb_smallint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_short_to_tsdb_int(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int16_t v = *(int16_t*)arg->val; + + return do_conv_int64_to_tsdb_int(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_short_to_tsdb_bigint(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + int16_t v = *(int16_t*)arg->val; + + return do_conv_int64_to_tsdb_bigint(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_double_to_tsdb_float(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + double v = *(double*)arg->val; + + return do_conv_double_to_tsdb_float(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_double_to_tsdb_double(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + double v = *(double*)arg->val; + + return do_conv_double_to_tsdb_double(stmt, v, arg); +} + +static SQLRETURN do_conv_sql_binary_to_tsdb_binary(stmt_t *stmt, tsdb_param_conv_arg_t *arg) { + unsigned char *buf = (unsigned char*)arg->val; + size_t len = (size_t)arg->soi; + OILE(len>0, ""); + + errs_t *errs = &stmt->errs; + + if (len > arg->tsdb_param->tsdb_bytes) { + SET_OOM(errs, "failed to convert binary for param [%d], binary too long [%zd/%d]", arg->idx+1, len, arg->tsdb_param->tsdb_bytes); + return SQL_ERROR; + } + + tsdb_param_val_t *tsdb_param_val = arg->tsdb_param_val; + TAOS_BIND *taos_bind = arg->taos_bind; + + taos_bind->buffer_type = TSDB_DATA_TYPE_BINARY; + taos_bind->u.bin = buf; + taos_bind->buffer_length = len; + taos_bind->buffer = taos_bind->u.bin; + taos_bind->length = &taos_bind->buffer_length; + taos_bind->is_null = &tsdb_param_val->is_null; + + return SQL_SUCCESS; +} + +static SQLRETURN do_set_param_wchar_conv_func(stmt_t *stmt, int idx, param_binding_t *binding, tsdb_param_t *tsdb_param) { + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + int tsdb_type = tsdb_param->tsdb_type; + + switch (tsdb_type) + { + case TSDB_DATA_TYPE_TIMESTAMP: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_timestamp; + } break; + case TSDB_DATA_TYPE_NCHAR: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_nchar; + } break; + case TSDB_DATA_TYPE_BOOL: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_bool; + } break; + case TSDB_DATA_TYPE_TINYINT: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_tinyint; + } break; + case TSDB_DATA_TYPE_SMALLINT: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_smallint; + } break; + case TSDB_DATA_TYPE_INT: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_int; + } break; + case TSDB_DATA_TYPE_BIGINT: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_bigint; + } break; + case TSDB_DATA_TYPE_FLOAT: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_float; + } break; + case TSDB_DATA_TYPE_DOUBLE: + { + tsdb_param->conv = do_conv_sql_wchar_to_tsdb_double; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for param [%d]", + valueType, sql_c_type(valueType), + tsdb_type, taos_data_type(tsdb_type), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_set_param_char_conv_func(stmt_t *stmt, int idx, param_binding_t *binding, tsdb_param_t *tsdb_param) { + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + int tsdb_type = tsdb_param->tsdb_type; + + switch (tsdb_type) + { + case TSDB_DATA_TYPE_TIMESTAMP: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_timestamp; + } break; + case TSDB_DATA_TYPE_NCHAR: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_nchar; + } break; + case TSDB_DATA_TYPE_BOOL: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_bool; + } break; + case TSDB_DATA_TYPE_TINYINT: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_tinyint; + } break; + case TSDB_DATA_TYPE_SMALLINT: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_smallint; + } break; + case TSDB_DATA_TYPE_INT: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_int; + } break; + case TSDB_DATA_TYPE_BIGINT: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_bigint; + } break; + case TSDB_DATA_TYPE_FLOAT: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_float; + } break; + case TSDB_DATA_TYPE_DOUBLE: + { + tsdb_param->conv = do_conv_sql_char_to_tsdb_double; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for param [%d]", + valueType, sql_c_type(valueType), + tsdb_type, taos_data_type(tsdb_type), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_set_param_sbigint_conv_func(stmt_t *stmt, int idx, param_binding_t *binding, tsdb_param_t *tsdb_param) { + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + int tsdb_type = tsdb_param->tsdb_type; + + switch (tsdb_type) + { + case TSDB_DATA_TYPE_BIGINT: + { + tsdb_param->conv = do_conv_sql_sbigint_to_tsdb_bigint; + } break; + case TSDB_DATA_TYPE_TINYINT: + { + tsdb_param->conv = do_conv_sql_sbigint_to_tsdb_tinyint; + } break; + case TSDB_DATA_TYPE_INT: + { + tsdb_param->conv = do_conv_sql_sbigint_to_tsdb_int; + } break; + case TSDB_DATA_TYPE_TIMESTAMP: + { + tsdb_param->conv = do_conv_sql_sbigint_to_tsdb_timestamp; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for param [%d]", + valueType, sql_c_type(valueType), + tsdb_type, taos_data_type(tsdb_type), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_set_param_long_conv_func(stmt_t *stmt, int idx, param_binding_t *binding, tsdb_param_t *tsdb_param) { + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + int tsdb_type = tsdb_param->tsdb_type; + + switch (tsdb_type) + { + case TSDB_DATA_TYPE_BOOL: + { + tsdb_param->conv = do_conv_sql_long_to_tsdb_bool; + } break; + case TSDB_DATA_TYPE_TINYINT: + { + tsdb_param->conv = do_conv_sql_long_to_tsdb_tinyint; + } break; + case TSDB_DATA_TYPE_SMALLINT: + { + tsdb_param->conv = do_conv_sql_long_to_tsdb_smallint; + } break; + case TSDB_DATA_TYPE_INT: + { + tsdb_param->conv = do_conv_sql_long_to_tsdb_int; + } break; + case TSDB_DATA_TYPE_BIGINT: + { + tsdb_param->conv = do_conv_sql_long_to_tsdb_bigint; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for param [%d]", + valueType, sql_c_type(valueType), + tsdb_type, taos_data_type(tsdb_type), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_set_param_tinyint_conv_func(stmt_t *stmt, int idx, param_binding_t *binding, tsdb_param_t *tsdb_param) { + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + int tsdb_type = tsdb_param->tsdb_type; + + switch (tsdb_type) + { + case TSDB_DATA_TYPE_BOOL: + { + tsdb_param->conv = do_conv_sql_tinyint_to_tsdb_bool; + } break; + case TSDB_DATA_TYPE_TINYINT: + { + tsdb_param->conv = do_conv_sql_tinyint_to_tsdb_tinyint; + } break; + case TSDB_DATA_TYPE_SMALLINT: + { + tsdb_param->conv = do_conv_sql_tinyint_to_tsdb_smallint; + } break; + case TSDB_DATA_TYPE_INT: + { + tsdb_param->conv = do_conv_sql_tinyint_to_tsdb_int; + } break; + case TSDB_DATA_TYPE_BIGINT: + { + tsdb_param->conv = do_conv_sql_tinyint_to_tsdb_bigint; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for param [%d]", + valueType, sql_c_type(valueType), + tsdb_type, taos_data_type(tsdb_type), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_set_param_short_conv_func(stmt_t *stmt, int idx, param_binding_t *binding, tsdb_param_t *tsdb_param) { + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + int tsdb_type = tsdb_param->tsdb_type; + + switch (tsdb_type) + { + case TSDB_DATA_TYPE_BOOL: + { + tsdb_param->conv = do_conv_sql_short_to_tsdb_bool; + } break; + case TSDB_DATA_TYPE_TINYINT: + { + tsdb_param->conv = do_conv_sql_short_to_tsdb_tinyint; + } break; + case TSDB_DATA_TYPE_SMALLINT: + { + tsdb_param->conv = do_conv_sql_short_to_tsdb_smallint; + } break; + case TSDB_DATA_TYPE_INT: + { + tsdb_param->conv = do_conv_sql_short_to_tsdb_int; + } break; + case TSDB_DATA_TYPE_BIGINT: + { + tsdb_param->conv = do_conv_sql_short_to_tsdb_bigint; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for param [%d]", + valueType, sql_c_type(valueType), + tsdb_type, taos_data_type(tsdb_type), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_set_param_double_conv_func(stmt_t *stmt, int idx, param_binding_t *binding, tsdb_param_t *tsdb_param) { + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + int tsdb_type = tsdb_param->tsdb_type; + + switch (tsdb_type) + { + case TSDB_DATA_TYPE_FLOAT: + { + tsdb_param->conv = do_conv_sql_double_to_tsdb_float; + } break; + case TSDB_DATA_TYPE_DOUBLE: + { + tsdb_param->conv = do_conv_sql_double_to_tsdb_double; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for param [%d]", + valueType, sql_c_type(valueType), + tsdb_type, taos_data_type(tsdb_type), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_set_param_binary_conv_func(stmt_t *stmt, int idx, param_binding_t *binding, tsdb_param_t *tsdb_param) { + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + int tsdb_type = tsdb_param->tsdb_type; + + switch (tsdb_type) + { + case TSDB_DATA_TYPE_BINARY: + { + tsdb_param->conv = do_conv_sql_binary_to_tsdb_binary; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for param [%d]", + valueType, sql_c_type(valueType), + tsdb_type, taos_data_type(tsdb_type), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + + +static SQLRETURN do_set_param_conv_func(stmt_t *stmt, int idx, param_binding_t *binding) { + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + tsdb_param_t *tsdb_params = tsdb_stmt->tsdb_params; + tsdb_param_t *tsdb_param = tsdb_params + idx; + + errs_t *errs = &stmt->errs; + + SQLSMALLINT valueType = binding->ValueType; + + switch (valueType) + { + case SQL_C_CHAR: + { + return do_set_param_char_conv_func(stmt, idx, binding, tsdb_param); + } break; + case SQL_C_WCHAR: + { + return do_set_param_wchar_conv_func(stmt, idx, binding, tsdb_param); + } break; + case SQL_C_SBIGINT: + { + return do_set_param_sbigint_conv_func(stmt, idx, binding, tsdb_param); + } break; + case SQL_C_LONG: + { + return do_set_param_long_conv_func(stmt, idx, binding, tsdb_param); + } break; + case SQL_C_TINYINT: + { + return do_set_param_tinyint_conv_func(stmt, idx, binding, tsdb_param); + } break; + case SQL_C_SHORT: + { + return do_set_param_short_conv_func(stmt, idx, binding, tsdb_param); + } break; + case SQL_C_DOUBLE: + { + return do_set_param_double_conv_func(stmt, idx, binding, tsdb_param); + } break; + case SQL_C_BINARY: + { + return do_set_param_binary_conv_func(stmt, idx, binding, tsdb_param); + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s for param [%d]", + valueType, sql_c_type(valueType), idx+1); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN tsdb_conn_set_param_conv(stmt_t *stmt, int idx) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt && tsdb_stmt->stmt==stmt, ""); + OILE(tsdb_stmt->tsdb_res==NULL, ""); + OILE(idx>=0, ""); + + paramset_t *paramset = &stmt->paramset; + param_t *params = paramset->params; + if (!params || idx>=paramset->n_params) return SQL_SUCCESS; + param_binding_t *bindings = paramset->bindings; + if (!bindings || idx>=paramset->n_bindings) return SQL_SUCCESS; + tsdb_param_t *tsdb_params = tsdb_stmt->tsdb_params; + OILE(tsdb_params, ""); + + param_binding_t *binding = bindings + idx; + + return do_set_param_conv_func(stmt, idx, binding); +} + +static SQLRETURN do_fill_param(stmt_t *stmt, int idx) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt && tsdb_stmt->stmt==stmt, ""); + OILE(tsdb_stmt->tsdb_res==NULL, ""); + + paramset_t *paramset = &stmt->paramset; + param_t *params = paramset->params; + tsdb_param_t *tsdb_params = tsdb_stmt->tsdb_params; + OILE(params, ""); + OILE(tsdb_params, ""); + OILE(idx>=0, ""); + OILE(idxn_params, ""); + param_t *param = params + idx; + tsdb_param_t *tsdb_param = tsdb_params + idx; + + errs_t *errs = &stmt->errs; + + int tsdb_type = 0; + int tsdb_bytes = 0; + int r = taos_stmt_get_param(tsdb_stmt->tsdb_stmt, idx, &tsdb_type, &tsdb_bytes); + if (r) { + SET_GENERAL(errs, "failed to get param[%d]", idx+1); + return SQL_ERROR; + } + + // https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/column-size-decimal-digits-transfer-octet-length-and-display-size?view=sql-server-ver15 + param->DecimalDigits = 0; + param->Nullable = SQL_NULLABLE; + tsdb_param->tsdb_type = tsdb_type; + tsdb_param->tsdb_bytes = tsdb_bytes; + switch (tsdb_type) + { + case TSDB_DATA_TYPE_TIMESTAMP: + { + param->DataType = SQL_CHAR; + param->ParameterSize = 23; + } break; + case TSDB_DATA_TYPE_NCHAR: + { + size_t bytes = ((size_t)tsdb_bytes - VARSTR_HEADER_SIZE); + size_t chars = bytes / TSDB_NCHAR_SIZE; + tsdb_param->tsdb_bytes = (int)bytes; + param->DataType = SQL_WCHAR; + param->ParameterSize = (SQLULEN)chars; + } break; + case TSDB_DATA_TYPE_BINARY: + { + size_t bytes = ((size_t)tsdb_bytes - VARSTR_HEADER_SIZE); + tsdb_param->tsdb_bytes = (int)bytes; + param->DataType = SQL_BINARY; + param->ParameterSize = (SQLULEN)bytes; + } break; + case TSDB_DATA_TYPE_BOOL: + { + param->DataType = SQL_BIT; + param->ParameterSize = 1; + } break; + case TSDB_DATA_TYPE_TINYINT: + { + param->DataType = SQL_TINYINT; + param->ParameterSize = 3; + } break; + case TSDB_DATA_TYPE_SMALLINT: + { + param->DataType = SQL_SMALLINT; + param->ParameterSize = 5; + } break; + case TSDB_DATA_TYPE_INT: + { + param->DataType = SQL_INTEGER; + param->ParameterSize = 10; + } break; + case TSDB_DATA_TYPE_BIGINT: + { + param->DataType = SQL_BIGINT; + param->ParameterSize = 20; + } break; + case TSDB_DATA_TYPE_FLOAT: + { + param->DataType = SQL_FLOAT; + param->ParameterSize = 15; + } break; + case TSDB_DATA_TYPE_DOUBLE: + { + param->DataType = SQL_DOUBLE; + param->ParameterSize = 15; + } break; + default: + { + SET_GENERAL(errs, "failed to map param[%d] type[%d]%s to SQL DATA TYPE", + idx+1, tsdb_type, taos_data_type(tsdb_type)); + return SQL_ERROR; + } break; + } + + param->ParameterNumber = (SQLUSMALLINT)(idx + 1); + + param_binding_t *bindings = paramset->bindings; + if (!bindings) return SQL_SUCCESS; + if (idx>=paramset->n_bindings) return SQL_SUCCESS; + + param_binding_t *binding = bindings + idx; + + return do_set_param_conv_func(stmt, idx, binding); +} + +static SQLRETURN tsdb_conn_prepare(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + OILE(stmt->prepared==0, ""); + OILE(STMT_IS_NORM(stmt), ""); + errs_t *errs = &stmt->errs; + + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + tsdb_conn_t *tsdb_conn = (tsdb_conn_t*)conn->ext.ext; + OILE(tsdb_conn && tsdb_conn->conn==conn, ""); + TAOS *taos = tsdb_conn->taos; + OILE(taos, ""); + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt && tsdb_stmt->stmt==stmt, ""); + OILE(tsdb_stmt->tsdb_res==NULL, ""); + // OILE(tsdb_stmt->tsdb_stmt==NULL, ""); + + tsdb_stmt->tsdb_params = NULL; + + if (!tsdb_stmt->tsdb_stmt) { + SQLRETURN r = tsdb_stmt_init_stmt(tsdb_stmt); + if (r!=SQL_SUCCESS) return r; + } + OILE(tsdb_stmt->tsdb_stmt, ""); + + tsdb_stmt_reclaim_param_vals(tsdb_stmt); + + do { + const char *txt = (const char*)stmt->sql.txt.buf; + size_t len = stmt->sql.txt.bytes; + OILE(txt, ""); + OILE(len>0, ""); + int r = taos_stmt_prepare(tsdb_stmt->tsdb_stmt, txt, (unsigned int)len); + if (r) { + SET_GENERAL(errs, "failed to prepare taos stmt:[%d]%s", r, tstrerror(r)); + break; + } + + int nums = 0; + r = taos_stmt_num_params(tsdb_stmt->tsdb_stmt, &nums); + if (r) { + SET_GENERAL(errs, "failed to prepare taos stmt:[%d]%s", r, tstrerror(r)); + break; + } + + paramset_t *paramset = &stmt->paramset; + OILE(paramset->params==NULL, ""); + OILE(paramset->n_params==0, ""); + OILE(tsdb_stmt->tsdb_params==NULL, ""); + + if (nums>0) { + paramset_init_params_cache(paramset); + tsdb_stmt_init_param_vals_cache(tsdb_stmt); + + if (!tsdb_stmt->tsdb_param_vals_cache) { + SET_OOM(errs, "failed to alloc val cache for params"); + return SQL_ERROR; + } + + todbc_buf_t *cache = stmt->paramset.params_cache; + if (!cache) { + SET_OOM(errs, "failed to alloc cache buffer for params"); + return SQL_ERROR; + } + OILE(cache, ""); + + param_t *params = (param_t*)todbc_buf_calloc(cache, (size_t)nums, sizeof(*params)); + if (!params) { + SET_OOM(errs, "failed to alloc buffer for params"); + return SQL_ERROR; + } + + tsdb_param_t *tsdb_params = (tsdb_param_t*)todbc_buf_calloc(cache, (size_t)nums, sizeof(*tsdb_params)); + if (!tsdb_params) { + SET_OOM(errs, "failed to alloc buffer for tsdb params"); + return SQL_ERROR; + } + + paramset->params = params; + paramset->n_params = nums; + tsdb_stmt->tsdb_params = tsdb_params; + + for (int i=0; iowner, ""); + + errs_t *errs = &stmt->errs; + + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + const char *enc_to = conn->enc_char; + const char *enc_from = conn->enc_src; // UTF8 + const unsigned char *src = (const unsigned char*)buf; + size_t slen = strlen(buf); + unsigned char *dst = (unsigned char*)col->TargetValue; + size_t dlen = (size_t)col->BufferLength; + todbc_string_t s = todbc_tls_write(enc_to, enc_from, src, &slen, dst, dlen); + if (!s.buf) { + SET_OOM(errs, "failed to convert timestamp"); + return SQL_ERROR; + } + OILE(s.bytes==s.total_bytes, ""); + if (col->StrLen_or_IndPtr) { + *col->StrLen_or_IndPtr = (SQLLEN)s.bytes; + } + for(size_t i=s.bytes; ierrs; + + switch (binding->TargetType) + { + case SQL_C_CHAR: + { + size_t len = (size_t)binding->BufferLength; + OILE(len>0, ""); + if (bytesTargetValue) { + memcpy(binding->TargetValue, val, len); + } + if (binding->StrLen_or_IndPtr) { + *binding->StrLen_or_IndPtr = (SQLLEN)len; + } + // do we really need this? + size_t dlen = (size_t)binding->BufferLength; + unsigned char *dst = (unsigned char*)binding->TargetValue; + for(size_t i=len; iBufferLength; + OILE(len>0, ""); + if (bytesTargetValue) { + memcpy(binding->TargetValue, val, len); + } + if (binding->StrLen_or_IndPtr) { + *binding->StrLen_or_IndPtr = (SQLLEN)len; + } + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for binding[%d]", + field->type, taos_data_type(field->type), + binding->TargetType, sql_c_type(binding->TargetType), + binding->ColumnNumber); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_nchar_to_sql_c(stmt_t *stmt, TAOS_FIELD *field, void *val, size_t bytes, col_binding_t *binding) { + errs_t *errs = &stmt->errs; + + OILE(stmt->owner, ""); + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + switch (binding->TargetType) + { + case SQL_C_CHAR: + { + const char *enc_to = conn->enc_char; + const char *enc_from = conn->enc_db; + const unsigned char *src = (const unsigned char*)val; + size_t slen = bytes; + unsigned char *dst = (unsigned char*)binding->TargetValue; + size_t dlen = (size_t)binding->BufferLength; + todbc_string_t s = todbc_tls_write(enc_to, enc_from, src, &slen, dst, dlen); + if (!s.buf) { + SET_OOM(errs, "failed to convert nchar"); + return SQL_ERROR; + } + OILE(s.bytes==s.total_bytes, ""); + if (binding->StrLen_or_IndPtr) { + *binding->StrLen_or_IndPtr = (SQLLEN)s.bytes; // com-on, it's NOT character-size + } + for(size_t i=s.bytes; ienc_wchar; + const char *enc_from = conn->enc_db; + const unsigned char *src = (const unsigned char*)val; + size_t slen = bytes; + unsigned char *dst = (unsigned char*)binding->TargetValue; + size_t dlen = (size_t)binding->BufferLength; + todbc_string_t s = todbc_tls_write(enc_to, enc_from, src, &slen, dst, dlen); + if (!s.buf) { + SET_OOM(errs, "failed to convert nchar"); + return SQL_ERROR; + } + OILE(s.bytes==s.total_bytes, ""); + if (binding->StrLen_or_IndPtr) { + *binding->StrLen_or_IndPtr = (SQLLEN)s.bytes; // com-on, it's NOT character-size + } + for(size_t i=s.bytes; itype, taos_data_type(field->type), + binding->TargetType, sql_c_type(binding->TargetType), + binding->ColumnNumber); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_int64_to_sql_c(stmt_t *stmt, TAOS_FIELD *field, int64_t val, col_binding_t *binding) { + errs_t *errs = &stmt->errs; + + char buf[128]; + + switch (binding->TargetType) + { + case SQL_C_CHAR: + { + snprintf(buf, sizeof(buf), "%" PRId64 "", val); + return do_conv_utf8_to_sql_c_char(stmt, buf, binding); + } break; + case SQL_C_UTINYINT: + { + if (val>UINT8_MAX || val<0) { + SET_GENERAL(errs, ""); + return SQL_ERROR; + } + if (binding->TargetValue) { + *(uint8_t*)binding->TargetValue = (uint8_t)val; + } + } break; + case SQL_C_USHORT: + { + if (val>UINT16_MAX || val<0) { + SET_GENERAL(errs, ""); + return SQL_ERROR; + } + if (binding->TargetValue) { + *(uint16_t*)binding->TargetValue = (uint16_t)val; + } + } break; + case SQL_C_SLONG: + { + if (val>INT32_MAX || valTargetValue) { + *(int32_t*)binding->TargetValue = (int32_t)val; + } + } break; + case SQL_C_UBIGINT: + { + if (binding->TargetValue) { + *(uint64_t*)binding->TargetValue = (uint64_t)val; + } + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for binding[%d]", + field->type, taos_data_type(field->type), + binding->TargetType, sql_c_type(binding->TargetType), + binding->ColumnNumber); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_timestamp_to_utf8(stmt_t *stmt, int64_t v, char *buf, size_t len) { + errs_t *errs = &stmt->errs; + + // microsecond precision, based on test + time_t secs = v / 1000; + int msecs = (int)(v % 1000); + + struct tm vtm = {0}; + if (&vtm!=localtime_r(&secs, &vtm)) { + SET_ERR(errs, "22007", "invalid timestamp"); + return SQL_ERROR; // ? SQL_SUCCESS_WITH_INFO + } + + char *p = buf; + size_t bytes = len; + + OILE(bytes>0, ""); + size_t n = strftime(p, bytes, "%Y-%m-%d %H:%M:%S", &vtm); + if (n==0) { + SET_GENERAL(errs, "failed to convert timestamp"); + return SQL_ERROR; // ? SQL_SUCCESS_WITH_INFO + } + p += n; + bytes -= n; + + OILE(bytes>0, ""); + int m = snprintf(p, bytes, ".%03d", msecs); + if (m>=bytes) { + SET_GENERAL(errs, "failed to convert timestamp"); + return SQL_ERROR; // ? SQL_SUCCESS_WITH_INFO + } + p += (size_t)m; + bytes -= (size_t)m; + + OILE(bytes>=0, ""); + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_timestamp_to_sql_c(stmt_t *stmt, TAOS_FIELD *field, int64_t val, col_binding_t *col) { + errs_t *errs = &stmt->errs; + + OILE(stmt, ""); + OILE(stmt->owner, ""); + + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + + SQLRETURN r; + char buf[128]; + + switch (col->TargetType) + { + case SQL_C_CHAR: + { + r = do_conv_timestamp_to_utf8(stmt, val, buf, sizeof(buf)); + if (r!=SQL_SUCCESS) return r; + return do_conv_utf8_to_sql_c_char(stmt, buf, col); + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for col[%d]", + field->type, taos_data_type(field->type), + col->TargetType, sql_c_type(col->TargetType), + col->ColumnNumber); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN do_conv_double_to_sql_c(stmt_t *stmt, TAOS_FIELD *field, double val, col_binding_t *binding) { + errs_t *errs = &stmt->errs; + + char buf[256]; + + switch (binding->TargetType) + { + case SQL_C_DOUBLE: + { + if (binding->TargetValue) { + *(double*)binding->TargetValue = val; + } + } break; + case SQL_C_FLOAT: + { + // shall we check overflow/underflow here? + if (binding->TargetValue) { + *(float*)binding->TargetValue = (float)val; + } + } break; + case SQL_C_CHAR: + { + snprintf(buf, sizeof(buf), "%lf", val); + return do_conv_utf8_to_sql_c_char(stmt, buf, binding); + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s to [%d] %s for binding[%d]", + field->type, taos_data_type(field->type), + binding->TargetType, sql_c_type(binding->TargetType), + binding->ColumnNumber); + return SQL_ERROR; + } break; + } + + return SQL_SUCCESS; +} + +static SQLRETURN tsdb_conn_proc_param(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt && tsdb_stmt->stmt==stmt, ""); + + paramset_t *paramset = &stmt->paramset; + param_t *params = paramset->params; + int n_params = paramset->n_params; + param_binding_t *bindings = paramset->bindings; + int n_bindings = paramset->n_bindings; + tsdb_param_t *tsdb_params = tsdb_stmt->tsdb_params; + int i_row = paramset->i_row; + int i_col = paramset->i_col; + + OILE(params && n_params>0, ""); + OILE(bindings && n_bindings>=0, ""); + OILE(tsdb_params, ""); + OILE(n_params==n_bindings, ""); + OILE(i_row>=0, ""); + OILE(i_col>=0 && i_colattr.bind_type; // default: SQL_PARAM_BIND_BY_COLUMN + // SQL_ATTR_PARAM_BIND_OFFSET_PTR + SQLULEN *bind_offset_ptr = stmt->attr.bind_offset_ptr; // default: NULL + // SQL_ATTR_PARAMSET_SIZE + SQLULEN paramset_size = stmt->attr.paramset_size; // default: 0 + + // OILE(bind_type && bind_type!=SQL_PARAM_BIND_BY_COLUMN, "[%ld]", bind_type); + // OILE(bind_offset_ptr, ""); + OILE(paramset_size>0, ""); + + OILE(i_rowParameterNumber>0 && param->ParameterNumber<=n_params, ""); + tsdb_param_t *tsdb_param = tsdb_params + i_col; + OILE(tsdb_param->conv, ""); + + tsdb_param_val_t *tsdb_param_vals = tsdb_stmt->tsdb_param_vals; + if (!tsdb_param_vals) { + errs_t *errs = &stmt->errs; + tsdb_stmt_calloc_param_vals(tsdb_stmt); + if (tsdb_stmt->tsdb_param_vals==NULL) { + SET_OOM(errs, "failed to alloc tsdb param vals for tsdb params"); + return SQL_ERROR; + } + if (tsdb_stmt->taos_binds==NULL) { + SET_OOM(errs, "failed to alloc taos binds for tsdb params"); + return SQL_ERROR; + } + tsdb_param_vals = tsdb_stmt->tsdb_param_vals; + } + OILE(tsdb_param_vals, ""); + TAOS_BIND *taos_binds = tsdb_stmt->taos_binds; + OILE(taos_binds, ""); + + + tsdb_param_val_t *tsdb_param_val = tsdb_param_vals + i_col; + tsdb_param_val->ParameterNumber = (SQLUSMALLINT)(i_col + 1); + tsdb_param_val->is_null = 1; + TAOS_BIND *taos_bind = taos_binds + i_col; + + param_binding_t *binding = bindings + (size_t)i_col; + OILE(binding->ParameterNumber==i_col+1, ""); + if (binding->ParameterValuePtr==NULL) return SQL_SUCCESS; + + SQLPOINTER val = binding->ParameterValuePtr; + SQLLEN *soip = binding->StrLen_or_IndPtr; + OILE(soip, ""); + + size_t offset = (size_t)i_row * (size_t)bind_type; + size_t bind_offset = 0; + if (bind_offset_ptr) bind_offset = *bind_offset_ptr; + + val = (SQLPOINTER)(((char*)val) + offset + bind_offset); + soip = (SQLLEN*)(((char*)soip) + offset + bind_offset); + + SQLLEN soi = *soip; + + if (soi == SQL_NULL_DATA) return SQL_SUCCESS; + + OILE(soi>=0 || soi==SQL_NTS, ""); + + tsdb_param_val->is_null = 0; + + conn_t *conn = stmt->owner->conn; + OILE(conn, ""); + todbc_buf_t *cache = tsdb_stmt->tsdb_param_vals_cache; + OILE(cache, ""); + + tsdb_param_conv_arg_t arg = { + .conn = conn, + .cache = cache, + .idx = i_col, + .val = val, + .soi = soi, + .tsdb_param = tsdb_param, + .tsdb_param_val = tsdb_param_val, + .taos_bind = taos_bind + }; + return tsdb_param->conv(stmt, &arg); +} + +static SQLRETURN tsdb_conn_param_row_processed(stmt_t *stmt) { + paramset_t *paramset = &stmt->paramset; + OILE(paramset->n_params>0, ""); + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt, ""); + TAOS_BIND *taos_binds = tsdb_stmt->taos_binds; + OILE(taos_binds, ""); + OILE(tsdb_stmt->tsdb_stmt, ""); + + errs_t *errs = &stmt->errs; + + if (1) { + int r = taos_stmt_bind_param(tsdb_stmt->tsdb_stmt, taos_binds); + if (r) { + SET_GENERAL(errs, "failed to bind params:[%d]%s", r, tstrerror(r)); + // keep executing/executed state unchanged + return SQL_ERROR; + } + + r = taos_stmt_add_batch(tsdb_stmt->tsdb_stmt); + if (r) { + SET_GENERAL(errs, "failed to add batch params:[%d]%s", r, tstrerror(r)); + // keep executing/executed state unchanged + return SQL_ERROR; + } + } + + return SQL_SUCCESS; +} + +static SQLRETURN tsdb_conn_execute(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + OILE(!STMT_IS_EXECUTED(stmt), ""); + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt, ""); + if (!tsdb_stmt->tsdb_stmt) { + SQLRETURN r = tsdb_stmt_init_stmt(tsdb_stmt); + if (r!=SQL_SUCCESS) return r; + OILE(0, ""); + } + OILE(tsdb_stmt->tsdb_stmt, ""); + + errs_t * errs = &stmt->errs; + + if (1) { + int r = 0; + + r = taos_stmt_execute(tsdb_stmt->tsdb_stmt); + if (r) { + SET_GENERAL(errs, "failed to execute:[%d]%s", r, tstrerror(r)); + // keep executing/executed state unchanged + return SQL_ERROR; + } + + tsdb_stmt->by_query = 0; + } + + STMT_SET_EXECUTED(stmt); + return SQL_SUCCESS; +} + +static void do_fetch_tsdb_res(tsdb_stmt_t *tsdb_stmt) { + if (!tsdb_stmt->tsdb_res) { + OILE(tsdb_stmt->by_query==0, ""); + OILE(tsdb_stmt->tsdb_stmt, ""); + tsdb_stmt->tsdb_res = taos_stmt_use_result(tsdb_stmt->tsdb_stmt); + OILE(tsdb_stmt->tsdb_res, ""); + // currently, TAOS_STMT does NOT co-exist with TAOS_RES + tsdb_stmt_close_stmt(tsdb_stmt); + } +} + +SQLRETURN tsdb_conn_get_affected_rows(stmt_t *stmt, SQLLEN *RowCount) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt, ""); + + do_fetch_tsdb_res(tsdb_stmt); + + int rows = taos_affected_rows(tsdb_stmt->tsdb_res); + OILE(RowCount, ""); + *RowCount = rows; + + return SQL_SUCCESS; +} + +static void do_fetch_tsdb_fields(tsdb_stmt_t *tsdb_stmt) { + if (!tsdb_stmt->tsdb_fields) { + tsdb_stmt->tsdb_fields = taos_fetch_fields(tsdb_stmt->tsdb_res); + } +} + +SQLRETURN tsdb_conn_get_fields_count(stmt_t *stmt, SQLSMALLINT *ColumnCount) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt, ""); + + do_fetch_tsdb_res(tsdb_stmt); + OILE(tsdb_stmt->tsdb_res, ""); + + int n_fields = taos_num_fields(tsdb_stmt->tsdb_res); + OILE(ColumnCount, ""); + *ColumnCount = (SQLSMALLINT)n_fields; + + return SQL_SUCCESS; +} + +SQLRETURN tsdb_conn_get_field(stmt_t *stmt, field_arg_t *arg) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + + errs_t *errs = &stmt->errs; + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt, ""); + + do_fetch_tsdb_res(tsdb_stmt); + OILE(tsdb_stmt->tsdb_res, ""); + + int n_fields = taos_num_fields(tsdb_stmt->tsdb_res); + OILE(arg->ColumnNumber>0, ""); + OILE(arg->ColumnNumber<=n_fields, ""); + do_fetch_tsdb_fields(tsdb_stmt); + OILE(tsdb_stmt->tsdb_fields, ""); + + TAOS_FIELD *field = tsdb_stmt->tsdb_fields + (arg->ColumnNumber-1); + int len = 0; + // charset ? + len = snprintf((char*)arg->ColumnName, (size_t)arg->BufferLength, "%s", field->name); + if (arg->NameLength) *arg->NameLength = (SQLSMALLINT)len; + if (arg->DecimalDigits) *arg->DecimalDigits = 0; + if (arg->Nullable) *arg->Nullable = SQL_NULLABLE; + + SQLSMALLINT DataType; + SQLULEN ColumnSize; + + // https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/column-size-decimal-digits-transfer-octet-length-and-display-size?view=sql-server-ver15 + switch (field->type) { + case TSDB_DATA_TYPE_TIMESTAMP: + { + DataType = SQL_CHAR; + ColumnSize = 23; + } break; + case TSDB_DATA_TYPE_NCHAR: + { + DataType = SQL_WCHAR; + ColumnSize = (SQLULEN)field->bytes; + } break; + case TSDB_DATA_TYPE_BINARY: + { + DataType = SQL_BINARY; + ColumnSize = (SQLULEN)field->bytes; + } break; + case TSDB_DATA_TYPE_BOOL: + { + DataType = SQL_BIT; + ColumnSize = 1; + } break; + case TSDB_DATA_TYPE_TINYINT: + { + DataType = SQL_TINYINT; + ColumnSize = 3; + } break; + case TSDB_DATA_TYPE_SMALLINT: + { + DataType = SQL_SMALLINT; + ColumnSize = 5; + } break; + case TSDB_DATA_TYPE_INT: + { + DataType = SQL_INTEGER; + ColumnSize = 10; + } break; + case TSDB_DATA_TYPE_BIGINT: + { + DataType = SQL_BIGINT; + ColumnSize = 20; + } break; + case TSDB_DATA_TYPE_FLOAT: + { + DataType = SQL_FLOAT; + ColumnSize = 15; + } break; + case TSDB_DATA_TYPE_DOUBLE: + { + DataType = SQL_DOUBLE; + ColumnSize = 15; + } break; + default: + { + SET_GENERAL(errs, "failed to map field[%d] type[%d]%s to SQL DATA TYPE", + arg->ColumnNumber, field->type, taos_data_type(field->type)); + return SQL_ERROR; + } break; + } + + if (arg->DataType) *arg->DataType = DataType; + if (arg->ColumnSize) *arg->ColumnSize = ColumnSize; + + return SQL_SUCCESS; +} + +SQLRETURN tsdb_conn_fetch(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt, ""); + + OILE(stmt->eof==0, ""); + + do_fetch_tsdb_res(tsdb_stmt); + OILE(tsdb_stmt->tsdb_res, ""); + + tsdb_stmt->tsdb_curr = taos_fetch_row(tsdb_stmt->tsdb_res); + if (!tsdb_stmt->tsdb_curr) { + stmt->eof = 1; + return SQL_NO_DATA; + } + + return SQL_SUCCESS; +} + +static SQLRETURN tsdb_conn_get_data(stmt_t *stmt, col_binding_t *col) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt, ""); + + OILE(stmt->eof==0, ""); + OILE(tsdb_stmt->tsdb_curr, ""); + OILE(tsdb_stmt->tsdb_res, ""); + + int n_fields = taos_num_fields(tsdb_stmt->tsdb_res); + int idx = (int)(col->ColumnNumber-1); + OILE(idx>=0, ""); + OILE(idxtsdb_fields, ""); + + TAOS_FIELD *field = tsdb_stmt->tsdb_fields + idx; + + OILE(col->StrLen_or_IndPtr, ""); + void *val = tsdb_stmt->tsdb_curr[idx]; + if (!val) { + *col->StrLen_or_IndPtr = SQL_NULL_DATA; + return SQL_SUCCESS; + } + + errs_t *errs = &stmt->errs; + + int64_t i64; + double dbl; + int is_dbl = 0; + + switch (field->type) + { + case TSDB_DATA_TYPE_TINYINT: + { + i64 = *(int8_t*)val; + } break; + case TSDB_DATA_TYPE_SMALLINT: + { + i64 = *(int16_t*)val; + } break; + case TSDB_DATA_TYPE_INT: + { + i64 = *(int32_t*)val; + } break; + case TSDB_DATA_TYPE_BIGINT: + { + i64 = *(int64_t*)val; + } break; + case TSDB_DATA_TYPE_FLOAT: + { + dbl = GET_FLOAT_VAL(val); + is_dbl = 1; + } break; + case TSDB_DATA_TYPE_DOUBLE: + { + dbl = GET_DOUBLE_VAL(val); + is_dbl = 1; + } break; + case TSDB_DATA_TYPE_BINARY: + { + size_t bytes = (size_t)varDataLen((char*)val - VARSTR_HEADER_SIZE); + return do_conv_binary_to_sql_c(stmt, field, val, bytes, col); + } break; + case TSDB_DATA_TYPE_NCHAR: + { + size_t bytes = (size_t)varDataLen((char*)val - VARSTR_HEADER_SIZE); + return do_conv_nchar_to_sql_c(stmt, field, val, bytes, col); + } break; + case TSDB_DATA_TYPE_TIMESTAMP: + { + i64 = *(int64_t*)val; + return do_conv_timestamp_to_sql_c(stmt, field, i64, col); + break; + } + case TSDB_DATA_TYPE_BOOL: + { + i64 = *(int8_t*)val; + } break; + default: + { + SET_GENERAL(errs, "not convertion from [%d]%s for col[%d]", + field->type, taos_data_type(field->type), col->ColumnNumber); + return SQL_ERROR; + } break; + } + if (is_dbl) return do_conv_double_to_sql_c(stmt, field, dbl, col); + else return do_conv_int64_to_sql_c(stmt, field, i64, col); +} + +static void tsdb_conn_close_rs(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->owner, ""); + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)stmt->ext.ext; + OILE(tsdb_stmt && tsdb_stmt->stmt==stmt, ""); + + tsdb_stmt_close_rs(tsdb_stmt); +} + +static int tsdb_conn_init_stmt(stmt_t *stmt) { + OILE(stmt, ""); + OILE(stmt->ext.ext==NULL, ""); + OILE(stmt->ext.free_stmt==NULL, ""); + + OILE(stmt->owner==NULL, ""); + + tsdb_stmt_t *tsdb_stmt = (tsdb_stmt_t*)calloc(1, sizeof(*tsdb_stmt)); + if (!tsdb_stmt) return -1; + + stmt_ext_t *ext = &stmt->ext; + + tsdb_stmt->stmt = stmt; + ext->ext = tsdb_stmt; + ext->free_stmt = tsdb_conn_free_stmt; + ext->clear_param_vals = tsdb_conn_clear_param_vals; + ext->exec_direct = tsdb_conn_exec_direct; + ext->prepare = tsdb_conn_prepare; + ext->set_param_conv = tsdb_conn_set_param_conv; + ext->proc_param = tsdb_conn_proc_param; + ext->param_row_processed = tsdb_conn_param_row_processed; + ext->execute = tsdb_conn_execute; + ext->get_affected_rows = tsdb_conn_get_affected_rows; + ext->get_fields_count = tsdb_conn_get_fields_count; + ext->get_field = tsdb_conn_get_field; + ext->fetch = tsdb_conn_fetch; + ext->get_data = tsdb_conn_get_data; + ext->close_rs = tsdb_conn_close_rs; + + return 0; +} + +static pthread_once_t init_once = PTHREAD_ONCE_INIT; +static int inited = 0; +// static char tsdb_svr_info[128] = ""; +// static char tsdb_cli_info[128] = ""; + +static void init_routine(void) { + int r = taos_init(); + if (r) { + OW("taos init failed: [%d]%s", r, tstrerror(r)); + return; + } + OI("taos inited"); + inited = 1; +} + +int conn_init_tsdb_conn(conn_t *conn) { + OILE(conn, ""); + OILE(conn->ext.ext==NULL, ""); + OILE(conn->ext.free_conn==NULL, ""); + + pthread_once(&init_once, init_routine); + if (!inited) return -1; + + tsdb_conn_t *tsdb_conn = (tsdb_conn_t*)calloc(1, sizeof(*tsdb_conn)); + if (!tsdb_conn) return -1; + + tsdb_conn->conn = conn; + conn->ext.ext = tsdb_conn; + conn->ext.free_conn = tsdb_conn_free; + conn->ext.connect = tsdb_conn_connect; + conn->ext.disconnect = tsdb_conn_disconnect; + conn->ext.init_stmt = tsdb_conn_init_stmt; + + return 0; +} + + diff --git a/src/connector/odbc/src/base/tsdb_impl.h b/src/connector/odbc/src/base/tsdb_impl.h new file mode 100644 index 0000000000000000000000000000000000000000..f217a4be410db8f69b82836e33c905481aa91ec8 --- /dev/null +++ b/src/connector/odbc/src/base/tsdb_impl.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _tsdb_impl_h_ +#define _tsdb_impl_h_ + +#include "../base.h" + +#include "conn.h" + +#define DEFAULT_SERVER "localhost:6030" + +int conn_init_tsdb_conn(conn_t *conn); + + +#endif // _tsdb_impl_h_ + + diff --git a/src/connector/odbc/src/col.h b/src/connector/odbc/src/col.h new file mode 100644 index 0000000000000000000000000000000000000000..f7cdc240813c5d96a6df0491132671e05fdfe374 --- /dev/null +++ b/src/connector/odbc/src/col.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _col_h_ +#define _col_h_ + +#include "base.h" + +struct col_s { +}; + + +#endif // _col_h_ + + + diff --git a/src/connector/odbc/src/install.cmd b/src/connector/odbc/src/install.cmd new file mode 100644 index 0000000000000000000000000000000000000000..4da8a2d9764686884d13318339ac5d1c3c8a8d15 --- /dev/null +++ b/src/connector/odbc/src/install.cmd @@ -0,0 +1,6 @@ +@echo off +REM install driver +odbcconf /A {INSTALLDRIVER "TAOS|Driver=C:\TDengine\driver\todbc.dll|ConnectFunctions=YYN|DriverODBCVer=03.00|FileUsage=0|SQLLevel=0"} +REM config user dsn +odbcconf /A {CONFIGDSN "TAOS" "DSN=TAOS_DSN"} + diff --git a/src/connector/odbc/src/install.sh b/src/connector/odbc/src/install.sh index 02f31de70ed76e150fbef5d388cbd8a3e9ba73b3..c08ac9208e56a8b4ead270d48825e13cd113a078 100755 --- a/src/connector/odbc/src/install.sh +++ b/src/connector/odbc/src/install.sh @@ -2,6 +2,13 @@ set -u +EXT="so" +[[ `uname` == 'Darwin' ]] && EXT="dylib" + +SUDO="sudo" +[[ `uname` == 'Darwin' ]] && SUDO="" + + BLD_DIR="$1" rm -f "${BLD_DIR}/template.ini" @@ -10,18 +17,30 @@ rm -f "${BLD_DIR}/template.dsn" cat > "${BLD_DIR}/template.ini" < "${BLD_DIR}/template.dsn" < - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// #define _BSD_SOURCE -#define _XOPEN_SOURCE -#define _DEFAULT_SOURCE -#define _GNU_SOURCE - -#include "todbc_log.h" -#include "todbc_flex.h" - -#include "taos.h" - -#include "tglobal.h" -#include "taoserror.h" -#include "todbc_util.h" -#include "todbc_conv.h" - -#include "os.h" - -#include -#include - -#ifndef FALSE -#define FALSE 0 -#endif - -#ifndef TRUE -#define TRUE 1 -#endif - -#define UTF8_ENC "UTF-8" -#define UTF16_ENC "UCS-2LE" -#define UNICODE_ENC "UCS-4LE" -#define GB18030_ENC "GB18030" - -#define GET_REF(obj) atomic_load_64(&obj->refcount) -#define INC_REF(obj) atomic_add_fetch_64(&obj->refcount, 1) -#define DEC_REF(obj) atomic_sub_fetch_64(&obj->refcount, 1) - -#define LOCK(obj) pthread_mutex_lock(&obj->lock); -#define UNLOCK(obj) pthread_mutex_unlock(&obj->lock); - -#define SET_ERROR(obj, sqlstate, eno, err_fmt, ...) \ -do { \ - obj->err.err_no = eno; \ - const char* estr = tstrerror(eno); \ - if (!estr) estr = "Unknown error"; \ - int n = snprintf(NULL, 0, "[TSDB:%x]%s: @%s[%d]" err_fmt "", \ - eno, estr, \ - basename((char*)__FILE__), __LINE__, \ - ##__VA_ARGS__); \ - if (n<0) break; \ - char *err_str = (char*)realloc(obj->err.err_str, (size_t)n+1); \ - if (!err_str) break; \ - obj->err.err_str = err_str; \ - snprintf(obj->err.err_str, (size_t)n+1, "[TSDB:%x]%s: @%s[%d]" err_fmt "", \ - eno, estr, \ - basename((char*)__FILE__), __LINE__, \ - ##__VA_ARGS__); \ - snprintf((char*)obj->err.sql_state, sizeof(obj->err.sql_state), "%s", sqlstate); \ -} while (0) - - -#define CLR_ERROR(obj) \ -do { \ - obj->err.err_no = TSDB_CODE_SUCCESS; \ - if (obj->err.err_str) obj->err.err_str[0] = '\0'; \ - obj->err.sql_state[0] = '\0'; \ -} while (0) - -#define FILL_ERROR(obj) \ -do { \ - size_t n = sizeof(obj->err.sql_state); \ - if (Sqlstate) strncpy((char*)Sqlstate, (char*)obj->err.sql_state, n); \ - if (NativeError) *NativeError = obj->err.err_no; \ - snprintf((char*)MessageText, (size_t)BufferLength, "%s", obj->err.err_str); \ - if (TextLength && obj->err.err_str) *TextLength = (SQLSMALLINT)utf8_chars(obj->err.err_str); \ -} while (0) - -#define FREE_ERROR(obj) \ -do { \ - obj->err.err_no = TSDB_CODE_SUCCESS; \ - if (obj->err.err_str) { \ - free(obj->err.err_str); \ - obj->err.err_str = NULL; \ - } \ - obj->err.sql_state[0] = '\0'; \ -} while (0) - -#define SET_UNSUPPORT_ERROR(obj, sqlstate, err_fmt, ...) \ -do { \ - SET_ERROR(obj, sqlstate, TSDB_CODE_ODBC_NOT_SUPPORT, err_fmt, ##__VA_ARGS__); \ -} while (0) \ - -#define SET_HANDLE_INVALID(obj, sqlstate, err_fmt, ...) \ -do { \ - SET_ERROR(obj, sqlstate, TSDB_CODE_QRY_INVALID_QHANDLE, err_fmt, ##__VA_ARGS__); \ -} while (0); - -#define SDUP(s,n) (s ? (s[(size_t)n] ? (const char*)strndup((const char*)s,(size_t)n) : (const char*)s) : strdup("")) -#define SFRE(x,s,n) \ -do { \ - if (x==(const char*)s) break; \ - if (x) { \ - free((char*)x); \ - x = NULL; \ - } \ -} while (0) - -#define CHK_CONN(obj) \ -do { \ - if (!obj->conn) { \ - SET_ERROR(obj, "HY000", TSDB_CODE_ODBC_INVALID_HANDLE, "connection closed or not ready"); \ - return SQL_ERROR; \ - } \ -} while (0); - -#define CHK_CONN_TAOS(obj) \ -do { \ - if (!obj->conn->taos) { \ - SET_ERROR(obj, "HY000", TSDB_CODE_ODBC_INVALID_HANDLE, "connection to data source closed or not ready"); \ - return SQL_ERROR; \ - } \ -} while (0); - -#define CHK_RS(r_091c, sql_091c, fmt_091c, ...) \ -do { \ - r_091c = SQL_ERROR; \ - int e = sql_091c->rs ? taos_errno(sql_091c->rs) : terrno; \ - if (e != TSDB_CODE_SUCCESS) { \ - SET_ERROR(sql_091c, "HY000", e, fmt_091c, ##__VA_ARGS__); \ - break; \ - } \ - r_091c = SQL_SUCCESS; \ -} while (0) - -#define NORM_STR_LENGTH(obj, ptr, len) \ -do { \ - if ((len) < 0 && (len)!=SQL_NTS) { \ - SET_ERROR((obj), "HY090", TSDB_CODE_ODBC_BAD_ARG, ""); \ - return SQL_ERROR; \ - } \ - if (len==SQL_NTS) len = (ptr) ? (SQLSMALLINT)strlen((const char*)(ptr)) : 0; \ -} while (0) - -#define PROFILING 0 - -#define PROFILE(statement) \ -do { \ - if (!PROFILING) { \ - statement; \ - break; \ - } \ - struct timeval tv0, tv1; \ - gettimeofday(&tv0, NULL); \ - statement; \ - gettimeofday(&tv1, NULL); \ - double delta = difftime(tv1.tv_sec, tv0.tv_sec); \ - delta *= 1000000; \ - delta += (double)(tv1.tv_usec-tv0.tv_usec); \ - delta /= 1000000; \ - D("%s: elapsed: [%.6f]s", #statement, delta); \ -} while (0) - -#define CHK_CONV(todb, statement) \ -do { \ - TSDB_CONV_CODE code_0c80 = (statement); \ - switch (code_0c80) { \ - case TSDB_CONV_OK: return SQL_SUCCESS; \ - case TSDB_CONV_OOM: \ - case TSDB_CONV_NOT_AVAIL: { \ - SET_ERROR(sql, "HY001", TSDB_CODE_ODBC_OOM, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_OOR: { \ - SET_ERROR(sql, "22003", TSDB_CODE_ODBC_CONV_OOR, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_CHAR_NOT_NUM: \ - case TSDB_CONV_CHAR_NOT_TS: { \ - SET_ERROR(sql, "22018", TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_NOT_VALID_TS: { \ - SET_ERROR(sql, "22007", TSDB_CODE_ODBC_CONV_NOT_VALID_TS, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_TRUNC_FRACTION: { \ - SET_ERROR(sql, "01S07", TSDB_CODE_ODBC_CONV_TRUNC_FRAC, ""); \ - return todb ? SQL_ERROR : SQL_SUCCESS_WITH_INFO; \ - } break; \ - case TSDB_CONV_TRUNC: { \ - SET_ERROR(sql, "22001", TSDB_CODE_ODBC_CONV_TRUNC, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_SRC_TOO_LARGE: { \ - SET_ERROR(sql, "22001", TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_SRC_BAD_SEQ: { \ - SET_ERROR(sql, "22001", TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_SRC_INCOMPLETE: { \ - SET_ERROR(sql, "22001", TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_SRC_GENERAL: { \ - SET_ERROR(sql, "22001", TSDB_CODE_ODBC_CONV_SRC_GENERAL, ""); \ - return SQL_ERROR; \ - } break; \ - case TSDB_CONV_BAD_CHAR: { \ - SET_ERROR(sql, "22001", TSDB_CODE_ODBC_CONV_TRUNC, ""); \ - return SQL_ERROR; \ - } break; \ - default: { \ - DASSERTX(0, "internal logic error: %d", code_0c80); \ - return SQL_ERROR; /* never reached here */ \ - } break; \ - } \ -} while (0) - -typedef struct env_s env_t; -typedef struct conn_s conn_t; -typedef struct sql_s sql_t; -typedef struct taos_error_s taos_error_t; -typedef struct param_bind_s param_bind_t; - -struct param_bind_s { - SQLUSMALLINT ParameterNumber; - SQLSMALLINT ValueType; - SQLSMALLINT ParameterType; - SQLULEN LengthPrecision; - SQLSMALLINT ParameterScale; - SQLPOINTER ParameterValue; - SQLLEN *StrLen_or_Ind; - - unsigned int valid; -}; - -struct taos_error_s { - char *err_str; - int err_no; - - SQLCHAR sql_state[6]; -}; - -struct env_s { - uint64_t refcount; - unsigned int destroying:1; - - char env_locale[64]; - char env_charset[64]; - - taos_error_t err; -}; - -struct conn_s { - uint64_t refcount; - env_t *env; - - char client_enc[64]; // ODBC client that communicates with this driver - char server_enc[64]; // taos dynamic library that's loaded by this driver - - tsdb_conv_t *client_to_server; - tsdb_conv_t *server_to_client; - tsdb_conv_t *utf8_to_client; - tsdb_conv_t *utf16_to_utf8; - tsdb_conv_t *utf16_to_server; - tsdb_conv_t *client_to_utf8; - - TAOS *taos; - - taos_error_t err; -}; - -struct sql_s { - uint64_t refcount; - conn_t *conn; - - TAOS_STMT *stmt; - param_bind_t *params; - int n_params; - size_t rowlen; - size_t n_rows; - size_t ptr_offset; - - TAOS_RES *rs; - TAOS_ROW row; - - taos_error_t err; - unsigned int is_prepared:1; - unsigned int is_insert:1; - unsigned int is_executed:1; -}; - -typedef struct c_target_s c_target_t; -struct c_target_s { - SQLUSMALLINT col; - SQLSMALLINT ct; // c type: SQL_C_XXX - char *ptr; - SQLLEN len; - SQLLEN *soi; -}; - -static pthread_once_t init_once = PTHREAD_ONCE_INIT; -static void init_routine(void); - -static size_t do_field_display_size(TAOS_FIELD *field); - -static tsdb_conv_t* tsdb_conn_client_to_server(conn_t *conn) { - if (!conn->client_to_server) { - conn->client_to_server = tsdb_conv_open(conn->client_enc, conn->server_enc); - } - return conn->client_to_server; -} - -static tsdb_conv_t* tsdb_conn_server_to_client(conn_t *conn) { - if (!conn->server_to_client) { - conn->server_to_client = tsdb_conv_open(conn->server_enc, conn->client_enc); - } - return conn->server_to_client; -} - -static tsdb_conv_t* tsdb_conn_utf8_to_client(conn_t *conn) { - if (!conn->utf8_to_client) { - conn->utf8_to_client = tsdb_conv_open(UTF8_ENC, conn->client_enc); - } - return conn->utf8_to_client; -} - -static tsdb_conv_t* tsdb_conn_utf16_to_utf8(conn_t *conn) { - if (!conn->utf16_to_utf8) { - conn->utf16_to_utf8 = tsdb_conv_open(UTF16_ENC, UTF8_ENC); - } - return conn->utf16_to_utf8; -} - -static tsdb_conv_t* tsdb_conn_utf16_to_server(conn_t *conn) { - if (!conn->utf16_to_server) { - conn->utf16_to_server = tsdb_conv_open(UTF16_ENC, conn->server_enc); - } - return conn->utf16_to_server; -} - -static tsdb_conv_t* tsdb_conn_client_to_utf8(conn_t *conn) { - if (!conn->client_to_utf8) { - conn->client_to_utf8 = tsdb_conv_open(conn->client_enc, UTF8_ENC); - } - return conn->client_to_utf8; -} - -static void tsdb_conn_close_convs(conn_t *conn) { - if (conn->client_to_server) { - tsdb_conv_close(conn->client_to_server); - conn->client_to_server = NULL; - } - if (conn->server_to_client) { - tsdb_conv_close(conn->server_to_client); - conn->server_to_client = NULL; - } - if (conn->utf8_to_client) { - tsdb_conv_close(conn->utf8_to_client); - conn->utf8_to_client = NULL; - } - if (conn->utf16_to_utf8) { - tsdb_conv_close(conn->utf16_to_utf8); - conn->utf16_to_utf8 = NULL; - } - if (conn->utf16_to_server) { - tsdb_conv_close(conn->utf16_to_server); - conn->utf16_to_server = NULL; - } - if (conn->client_to_utf8) { - tsdb_conv_close(conn->client_to_utf8); - conn->client_to_utf8 = NULL; - } -} - -#define SFREE(buffer, v, src) \ -do { \ - const char *v_096a = (const char*)(v); \ - const char *src_6a = (const char*)(src); \ - if (v_096a && v_096a!=src_6a && !is_owned_by_stack_buffer((buffer), v_096a)) { \ - free((char*)v_096a); \ - } \ -} while (0) - -static SQLRETURN doSQLAllocEnv(SQLHENV *EnvironmentHandle) -{ - pthread_once(&init_once, init_routine); - - env_t *env = (env_t*)calloc(1, sizeof(*env)); - if (!env) return SQL_INVALID_HANDLE; - - DASSERT(INC_REF(env)>0); - - snprintf(env->env_locale, sizeof(env->env_locale), "%s", tsLocale); - snprintf(env->env_charset, sizeof(env->env_charset), "%s", tsCharset); - - *EnvironmentHandle = env; - - CLR_ERROR(env); - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLAllocEnv(SQLHENV *EnvironmentHandle) -{ - SQLRETURN r; - r = doSQLAllocEnv(EnvironmentHandle); - return r; -} - -static SQLRETURN doSQLFreeEnv(SQLHENV EnvironmentHandle) -{ - env_t *env = (env_t*)EnvironmentHandle; - if (!env) return SQL_INVALID_HANDLE; - - DASSERT(GET_REF(env)==1); - - DASSERT(!env->destroying); - - env->destroying = 1; - DASSERT(env->destroying == 1); - - DASSERT(DEC_REF(env)==0); - - FREE_ERROR(env); - free(env); - - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLFreeEnv(SQLHENV EnvironmentHandle) -{ - SQLRETURN r; - r = doSQLFreeEnv(EnvironmentHandle); - return r; -} - -static SQLRETURN doSQLAllocConnect(SQLHENV EnvironmentHandle, - SQLHDBC *ConnectionHandle) -{ - env_t *env = (env_t*)EnvironmentHandle; - if (!env) return SQL_INVALID_HANDLE; - - if (!ConnectionHandle) { - SET_ERROR(env, "HY009", TSDB_CODE_ODBC_BAD_ARG, "ConnectionHandle [%p] not valid", ConnectionHandle); - return SQL_ERROR; - } - - DASSERT(INC_REF(env)>1); - - conn_t *conn = NULL; - do { - conn = (conn_t*)calloc(1, sizeof(*conn)); - if (!conn) { - SET_ERROR(env, "HY001", TSDB_CODE_ODBC_OOM, ""); - break; - } - - conn->env = env; - - snprintf(conn->client_enc, sizeof(conn->client_enc), "%s", conn->env->env_charset); - snprintf(conn->server_enc, sizeof(conn->server_enc), "%s", conn->env->env_charset); - - *ConnectionHandle = conn; - - DASSERT(INC_REF(conn)>0); - - return SQL_SUCCESS; - } while (0); - - DASSERT(DEC_REF(env)>0); - - return SQL_ERROR; -} - -SQLRETURN SQL_API SQLAllocConnect(SQLHENV EnvironmentHandle, - SQLHDBC *ConnectionHandle) -{ - SQLRETURN r; - r = doSQLAllocConnect(EnvironmentHandle, ConnectionHandle); - return r; -} - -static SQLRETURN doSQLFreeConnect(SQLHDBC ConnectionHandle) -{ - conn_t *conn = (conn_t*)ConnectionHandle; - if (!conn) return SQL_INVALID_HANDLE; - - DASSERT(GET_REF(conn)==1); - - DASSERT(conn->env); - - do { - if (conn->taos) { - taos_close(conn->taos); - conn->taos = NULL; - } - - DASSERT(DEC_REF(conn->env)>0); - DASSERT(DEC_REF(conn)==0); - - conn->env = NULL; - FREE_ERROR(conn); - tsdb_conn_close_convs(conn); - free(conn); - } while (0); - - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLFreeConnect(SQLHDBC ConnectionHandle) -{ - SQLRETURN r; - r = doSQLFreeConnect(ConnectionHandle); - return r; -} - -static SQLRETURN doSQLConnect(SQLHDBC ConnectionHandle, - SQLCHAR *ServerName, SQLSMALLINT NameLength1, - SQLCHAR *UserName, SQLSMALLINT NameLength2, - SQLCHAR *Authentication, SQLSMALLINT NameLength3) -{ - stack_buffer_t buffer; buffer.next = 0; - - conn_t *conn = (conn_t*)ConnectionHandle; - if (!conn) return SQL_ERROR; - - if (conn->taos) { - SET_ERROR(conn, "08002", TSDB_CODE_ODBC_CONNECTION_BUSY, "connection still in use"); - return SQL_ERROR; - } - - NORM_STR_LENGTH(conn, ServerName, NameLength1); - NORM_STR_LENGTH(conn, UserName, NameLength2); - NORM_STR_LENGTH(conn, Authentication, NameLength3); - - if (NameLength1>SQL_MAX_DSN_LENGTH) { - SET_ERROR(conn, "HY090", TSDB_CODE_ODBC_BAD_ARG, ""); - return SQL_ERROR; - } - - tsdb_conv_t *client_to_server = tsdb_conn_client_to_server(conn); - const char *dsn = NULL; - const char *uid = NULL; - const char *pwd = NULL; - const char *svr = NULL; - char server[4096]; server[0] = '\0'; - - do { - tsdb_conv(client_to_server, &buffer, (const char*)ServerName, (size_t)NameLength1, &dsn, NULL); - tsdb_conv(client_to_server, &buffer, (const char*)UserName, (size_t)NameLength2, &uid, NULL); - tsdb_conv(client_to_server, &buffer, (const char*)Authentication, (size_t)NameLength3, &pwd, NULL); - int n = SQLGetPrivateProfileString(dsn, "Server", "", server, sizeof(server)-1, "Odbc.ini"); - if (n<=0) { - snprintf(server, sizeof(server), "localhost:6030"); // all 7-bit ascii - } - tsdb_conv(client_to_server, &buffer, (const char*)server, (size_t)strlen(server), &svr, NULL); - - if ((!dsn) || (!uid) || (!pwd) || (!svr)) { - SET_ERROR(conn, "HY001", TSDB_CODE_ODBC_OOM, ""); - break; - } - - char *ip = NULL; - int port = 0; - char *p = strchr(svr, ':'); - if (p) { - ip = strndup(svr, (size_t)(p-svr)); - port = atoi(p+1); - } - - // TODO: data-race - // TODO: shall receive ip/port from odbc.ini - conn->taos = taos_connect(ip, uid, pwd, NULL, (uint16_t)port); - if (!conn->taos) { - SET_ERROR(conn, "08001", terrno, "failed to connect to data source for DSN[%s] @[%s:%d]", dsn, ip, port); - break; - } - } while (0); - - tsdb_conv_free(client_to_server, dsn, &buffer, (const char*)ServerName); - tsdb_conv_free(client_to_server, uid, &buffer, (const char*)UserName); - tsdb_conv_free(client_to_server, pwd, &buffer, (const char*)Authentication); - tsdb_conv_free(client_to_server, svr, &buffer, (const char*)server); - - return conn->taos ? SQL_SUCCESS : SQL_ERROR; -} - -SQLRETURN SQL_API SQLConnect(SQLHDBC ConnectionHandle, - SQLCHAR *ServerName, SQLSMALLINT NameLength1, - SQLCHAR *UserName, SQLSMALLINT NameLength2, - SQLCHAR *Authentication, SQLSMALLINT NameLength3) -{ - SQLRETURN r; - r = doSQLConnect(ConnectionHandle, ServerName, NameLength1, - UserName, NameLength2, - Authentication, NameLength3); - return r; -} - -static SQLRETURN doSQLDisconnect(SQLHDBC ConnectionHandle) -{ - conn_t *conn = (conn_t*)ConnectionHandle; - if (!conn) return SQL_INVALID_HANDLE; - - if (conn->taos) { - taos_close(conn->taos); - conn->taos = NULL; - } - - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLDisconnect(SQLHDBC ConnectionHandle) -{ - SQLRETURN r; - r = doSQLDisconnect(ConnectionHandle); - return r; -} - -static SQLRETURN doSQLAllocStmt(SQLHDBC ConnectionHandle, - SQLHSTMT *StatementHandle) -{ - conn_t *conn = (conn_t*)ConnectionHandle; - if (!conn) return SQL_INVALID_HANDLE; - - if (!StatementHandle) { - SET_ERROR(conn, "HY009", TSDB_CODE_ODBC_BAD_ARG, "StatementHandle [%p] not valid", StatementHandle); - return SQL_ERROR; - } - - DASSERT(INC_REF(conn)>1); - - do { - sql_t *sql = (sql_t*)calloc(1, sizeof(*sql)); - if (!sql) { - SET_ERROR(conn, "HY001", TSDB_CODE_ODBC_OOM, ""); - break; - } - - sql->conn = conn; - DASSERT(INC_REF(sql)>0); - - *StatementHandle = sql; - - return SQL_SUCCESS; - } while (0); - - DASSERT(DEC_REF(conn)>0); - - return SQL_ERROR; -} - -SQLRETURN SQL_API SQLAllocStmt(SQLHDBC ConnectionHandle, - SQLHSTMT *StatementHandle) -{ - SQLRETURN r; - r = doSQLAllocStmt(ConnectionHandle, StatementHandle); - return r; -} - -static SQLRETURN doSQLAllocHandle(SQLSMALLINT HandleType, SQLHANDLE InputHandle, SQLHANDLE *OutputHandle) -{ - switch (HandleType) { - case SQL_HANDLE_ENV: { - SQLHENV env = {0}; - if (!OutputHandle) return SQL_ERROR; - SQLRETURN r = doSQLAllocEnv(&env); - if (r==SQL_SUCCESS) *OutputHandle = env; - return r; - } break; - case SQL_HANDLE_DBC: { - SQLRETURN r = doSQLAllocConnect(InputHandle, OutputHandle); - return r; - } break; - case SQL_HANDLE_STMT: { - SQLRETURN r = doSQLAllocStmt(InputHandle, OutputHandle); - return r; - } break; - default: { - return SQL_ERROR; - } break; - } -} - -SQLRETURN SQL_API SQLAllocHandle(SQLSMALLINT HandleType, SQLHANDLE InputHandle, SQLHANDLE *OutputHandle) -{ - SQLRETURN r; - r = doSQLAllocHandle(HandleType, InputHandle, OutputHandle); - return r; -} - -static SQLRETURN doSQLFreeStmt(SQLHSTMT StatementHandle, - SQLUSMALLINT Option) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - switch (Option) { - case SQL_CLOSE: return SQL_SUCCESS; - case SQL_DROP: break; - case SQL_UNBIND: - case SQL_RESET_PARAMS: { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NOT_SUPPORT, "free statement with Option[%x] not supported yet", Option); - return SQL_ERROR; - } break; - default: { - SET_ERROR(sql, "HY092", TSDB_CODE_ODBC_OUT_OF_RANGE, "free statement with Option[%x] not supported yet", Option); - return SQL_ERROR; - } break; - } - - DASSERT(GET_REF(sql)==1); - - if (sql->rs) { - taos_free_result(sql->rs); - sql->rs = NULL; - } - - if (sql->stmt) { - taos_stmt_close(sql->stmt); - sql->stmt = NULL; - } - - if (sql->params) { - free(sql->params); - sql->params = NULL; - } - sql->n_params = 0; - - DASSERT(DEC_REF(sql->conn)>0); - DASSERT(DEC_REF(sql)==0); - - sql->conn = NULL; - - FREE_ERROR(sql); - - free(sql); - - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLFreeStmt(SQLHSTMT StatementHandle, - SQLUSMALLINT Option) -{ - SQLRETURN r; - r = doSQLFreeStmt(StatementHandle, Option); - return r; -} - -static SQLRETURN do_exec_direct(sql_t *sql, TSDB_CONV_CODE code, const char *statement) { - if (code) CHK_CONV(1, code); - DASSERT(code==TSDB_CONV_OK); - - SQLRETURN r = SQL_ERROR; - do { - sql->rs = taos_query(sql->conn->taos, statement); - CHK_RS(r, sql, "failed to execute"); - } while (0); - - return r; -} - -static SQLRETURN doSQLExecDirect(SQLHSTMT StatementHandle, - SQLCHAR *StatementText, SQLINTEGER TextLength) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - conn_t *conn = sql->conn; - - NORM_STR_LENGTH(sql, StatementText, TextLength); - - if (sql->rs) { - taos_free_result(sql->rs); - sql->rs = NULL; - sql->row = NULL; - } - - if (sql->stmt) { - taos_stmt_close(sql->stmt); - sql->stmt = NULL; - } - - if (sql->params) { - free(sql->params); - sql->params = NULL; - } - sql->n_params = 0; - - SQLRETURN r = SQL_SUCCESS; - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_server = tsdb_conn_client_to_server(conn); - const char *stxt = NULL; - do { - TSDB_CONV_CODE code = tsdb_conv(client_to_server, &buffer, (const char*)StatementText, (size_t)TextLength, &stxt, NULL); - r = do_exec_direct(sql, code, stxt); - } while (0); - tsdb_conv_free(client_to_server, stxt, &buffer, (const char*)StatementText); - - return r; -} - -SQLRETURN SQL_API SQLExecDirect(SQLHSTMT StatementHandle, - SQLCHAR *StatementText, SQLINTEGER TextLength) -{ - SQLRETURN r; - r = doSQLExecDirect(StatementHandle, StatementText, TextLength); - return r; -} - -static SQLRETURN doSQLExecDirectW(SQLHSTMT hstmt, SQLWCHAR *szSqlStr, SQLINTEGER cbSqlStr) -{ - sql_t *sql = (sql_t*)hstmt; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - conn_t *conn = sql->conn; - - if (!szSqlStr) { - SET_ERROR(sql, "HY009", TSDB_CODE_ODBC_BAD_ARG, "szSqlStr [%p] not allowed", szSqlStr); - return SQL_ERROR; - } - if (cbSqlStr < 0) { - SET_ERROR(sql, "HY090", TSDB_CODE_ODBC_BAD_ARG, ""); - return SQL_ERROR; - } - - SQLRETURN r = SQL_SUCCESS; - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_server = tsdb_conn_utf16_to_server(conn); - const char *stxt = NULL; - do { - size_t slen = (size_t)cbSqlStr * sizeof(*szSqlStr); - TSDB_CONV_CODE code = tsdb_conv(utf16_to_server, &buffer, (const char*)szSqlStr, slen, &stxt, NULL); - r = do_exec_direct(sql, code, stxt); - } while (0); - tsdb_conv_free(utf16_to_server, stxt, &buffer, (const char*)szSqlStr); - - return r; -} - -SQLRETURN SQL_API SQLExecDirectW(SQLHSTMT hstmt, SQLWCHAR *szSqlStr, SQLINTEGER cbSqlStr) -{ - SQLRETURN r = doSQLExecDirectW(hstmt, szSqlStr, cbSqlStr); - return r; -} - -static SQLRETURN doSQLNumResultCols(SQLHSTMT StatementHandle, - SQLSMALLINT *ColumnCount) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - if (sql->is_insert) { - if (ColumnCount) { - *ColumnCount = 0; - } - return SQL_SUCCESS; - } - - if (!sql->rs) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NO_RESULT, ""); - return SQL_ERROR; - } - - int fields = taos_field_count(sql->rs); - if (ColumnCount) { - *ColumnCount = (SQLSMALLINT)fields; - } - - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLNumResultCols(SQLHSTMT StatementHandle, - SQLSMALLINT *ColumnCount) -{ - SQLRETURN r; - r = doSQLNumResultCols(StatementHandle, ColumnCount); - return r; -} - -static SQLRETURN doSQLRowCount(SQLHSTMT StatementHandle, - SQLLEN *RowCount) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_ERROR; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - // ref: https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlrowcount-function?view=sql-server-ver15 - // Summary - // SQLRowCount returns the number of rows affected by an UPDATE, INSERT, or DELETE statement; - // an SQL_ADD, SQL_UPDATE_BY_BOOKMARK, or SQL_DELETE_BY_BOOKMARK operation in SQLBulkOperations; - // or an SQL_UPDATE or SQL_DELETE operation in SQLSetPos. - - // how to fetch affected rows from taos? - // taos_affected_rows? - - if (1) { - SET_ERROR(sql, "IM001", TSDB_CODE_ODBC_NOT_SUPPORT, ""); - // if (RowCount) *RowCount = 0; - return SQL_SUCCESS_WITH_INFO; - } - - if (!sql->is_insert) { - if (RowCount) *RowCount = 0; - return SQL_SUCCESS; - } - - if (!sql->rs) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NO_RESULT, ""); - return SQL_ERROR; - } - - int rows = taos_affected_rows(sql->rs); - if (RowCount) { - *RowCount = rows; - } - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLRowCount(SQLHSTMT StatementHandle, - SQLLEN *RowCount) -{ - SQLRETURN r; - r = doSQLRowCount(StatementHandle, RowCount); - return r; -} - -static SQLRETURN doSQLColAttribute(SQLHSTMT StatementHandle, - SQLUSMALLINT ColumnNumber, SQLUSMALLINT FieldIdentifier, - SQLPOINTER CharacterAttribute, SQLSMALLINT BufferLength, - SQLSMALLINT *StringLength, SQLLEN *NumericAttribute ) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_ERROR; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - if (!sql->rs) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NO_RESULT, ""); - return SQL_ERROR; - } - - int nfields = taos_field_count(sql->rs); - TAOS_FIELD *fields = taos_fetch_fields(sql->rs); - - if (nfields==0 || fields==NULL) { - SET_ERROR(sql, "07005", TSDB_CODE_ODBC_NO_FIELDS, ""); - return SQL_ERROR; - } - - if (ColumnNumber<=0 || ColumnNumber>nfields) { - SET_ERROR(sql, "07009", TSDB_CODE_ODBC_OUT_OF_RANGE, "invalid column number [%d]", ColumnNumber); - return SQL_ERROR; - } - - TAOS_FIELD *field = fields + ColumnNumber-1; - - switch (FieldIdentifier) { - case SQL_COLUMN_DISPLAY_SIZE: { - *NumericAttribute = (SQLLEN)do_field_display_size(field); - } break; - case SQL_COLUMN_LABEL: { - // todo: check BufferLength - size_t n = sizeof(field->name); - strncpy(CharacterAttribute, field->name, (n>BufferLength ? (size_t)BufferLength : n)); - } break; - case SQL_COLUMN_UNSIGNED: { - *NumericAttribute = SQL_FALSE; - } break; - default: { - SET_ERROR(sql, "HY091", TSDB_CODE_ODBC_OUT_OF_RANGE, - "FieldIdentifier[%d/0x%x] for Column [%d] not supported yet", - FieldIdentifier, FieldIdentifier, ColumnNumber); - return SQL_ERROR; - } break; - } - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLColAttribute(SQLHSTMT StatementHandle, - SQLUSMALLINT ColumnNumber, SQLUSMALLINT FieldIdentifier, - SQLPOINTER CharacterAttribute, SQLSMALLINT BufferLength, - SQLSMALLINT *StringLength, SQLLEN *NumericAttribute ) -{ - SQLRETURN r; - r = doSQLColAttribute(StatementHandle, ColumnNumber, FieldIdentifier, - CharacterAttribute, BufferLength, - StringLength, NumericAttribute); - return r; -} - -static SQLRETURN doSQLGetData(SQLHSTMT StatementHandle, - SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, - SQLPOINTER TargetValue, SQLLEN BufferLength, - SQLLEN *StrLen_or_Ind) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - conn_t *conn = sql->conn; - - if (!sql->rs) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NO_RESULT, ""); - return SQL_ERROR; - } - - if (!sql->row) { - SET_ERROR(sql, "24000", TSDB_CODE_ODBC_INVALID_CURSOR, ""); - return SQL_ERROR; - } - - int nfields = taos_field_count(sql->rs); - TAOS_FIELD *fields = taos_fetch_fields(sql->rs); - - if (ColumnNumber<=0 || ColumnNumber>nfields) { - SET_ERROR(sql, "07009", TSDB_CODE_ODBC_OUT_OF_RANGE, "invalid column number [%d]", ColumnNumber); - return SQL_ERROR; - } - - if (TargetValue == NULL) { - SET_ERROR(sql, "HY009", TSDB_CODE_ODBC_BAD_ARG, "NULL TargetValue not allowed for col [%d]", ColumnNumber); - return SQL_ERROR; - } - if (BufferLength<0) { - SET_ERROR(sql, "HY090", TSDB_CODE_ODBC_BAD_ARG, ""); - return SQL_ERROR; - } - - TAOS_FIELD *field = fields + ColumnNumber-1; - void *row = sql->row[ColumnNumber-1]; - - if (!row) { - if (!StrLen_or_Ind) { - SET_ERROR(sql, "22002", TSDB_CODE_ODBC_BAD_ARG, "NULL StrLen_or_Ind not allowed for col [%d]", ColumnNumber); - return SQL_ERROR; - } - *StrLen_or_Ind = SQL_NULL_DATA; - return SQL_SUCCESS; - } - - c_target_t target = {0}; - target.col = ColumnNumber; - target.ct = TargetType; - target.ptr = TargetValue; - target.len = BufferLength; - target.soi = StrLen_or_Ind; - - switch (field->type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: { - int64_t v; - switch (field->type) { - case TSDB_DATA_TYPE_BOOL: v = *(int8_t*)row; if (v) v = 1; break; - case TSDB_DATA_TYPE_TINYINT: v = *(int8_t*)row; break; - case TSDB_DATA_TYPE_SMALLINT: v = *(int16_t*)row; break; - case TSDB_DATA_TYPE_INT: v = *(int32_t*)row; break; - case TSDB_DATA_TYPE_BIGINT: // fall through - default: v = *(int64_t*)row; break; - } - switch (target.ct) { - case SQL_C_BIT: { - CHK_CONV(0, tsdb_int64_to_bit(v, TargetValue)); - } break; - case SQL_C_TINYINT: { - CHK_CONV(0, tsdb_int64_to_tinyint(v, TargetValue)); - } break; - case SQL_C_SHORT: { - CHK_CONV(0, tsdb_int64_to_smallint(v, TargetValue)); - } break; - case SQL_C_LONG: { - CHK_CONV(0, tsdb_int64_to_int(v, TargetValue)); - } break; - case SQL_C_SBIGINT: { - CHK_CONV(0, tsdb_int64_to_bigint(v, TargetValue)); - } break; - case SQL_C_FLOAT: { - CHK_CONV(0, tsdb_int64_to_float(v, TargetValue)); - } break; - case SQL_C_DOUBLE: { - CHK_CONV(0, tsdb_int64_to_double(v, TargetValue)); - } break; - case SQL_C_CHAR: { - tsdb_conv_t *utf8_to_client = tsdb_conn_utf8_to_client(conn); - size_t len = (size_t)BufferLength; - TSDB_CONV_CODE code = tsdb_conv_write_int64(utf8_to_client, v, (char*)TargetValue, &len); - if (StrLen_or_Ind) *StrLen_or_Ind = (SQLLEN)len; - CHK_CONV(0, code); - } break; - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, - "no convertion from [%s] to [%s[%d][0x%x]] for col [%d]", - taos_data_type(field->type), sql_c_type(target.ct), target.ct, target.ct, ColumnNumber); - return SQL_ERROR; - } - } - } break; - case TSDB_DATA_TYPE_FLOAT: { - float v = *(float*)row; - switch (target.ct) { - case SQL_C_FLOAT: { - *(float*)TargetValue = v; - return SQL_SUCCESS; - } break; - case SQL_C_DOUBLE: { - *(double*)TargetValue = v; - return SQL_SUCCESS; - } break; - case SQL_C_CHAR: { - tsdb_conv_t *utf8_to_client = tsdb_conn_utf8_to_client(conn); - size_t len = (size_t)BufferLength; - TSDB_CONV_CODE code = tsdb_conv_write_double(utf8_to_client, v, (char*)TargetValue, &len); - if (StrLen_or_Ind) *StrLen_or_Ind = (SQLLEN)len; - CHK_CONV(0, code); - } break; - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, - "no convertion from [%s] to [%s[%d][0x%x]] for col [%d]", - taos_data_type(field->type), sql_c_type(target.ct), target.ct, target.ct, ColumnNumber); - return SQL_ERROR; - } - } - } break; - case TSDB_DATA_TYPE_DOUBLE: { - double v = *(double*)row; - switch (target.ct) { - case SQL_C_DOUBLE: { - *(double*)TargetValue = v; - return SQL_SUCCESS; - } break; - case SQL_C_CHAR: { - tsdb_conv_t *utf8_to_client = tsdb_conn_utf8_to_client(conn); - size_t len = (size_t)BufferLength; - TSDB_CONV_CODE code = tsdb_conv_write_double(utf8_to_client, v, (char*)TargetValue, &len); - if (StrLen_or_Ind) *StrLen_or_Ind = (SQLLEN)len; - CHK_CONV(0, code); - } break; - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, - "no convertion from [%s] to [%s[%d][0x%x]] for col [%d]", - taos_data_type(field->type), sql_c_type(target.ct), target.ct, target.ct, ColumnNumber); - return SQL_ERROR; - } - } - } break; - case TSDB_DATA_TYPE_TIMESTAMP: { - SQL_TIMESTAMP_STRUCT ts = {0}; - int64_t v = *(int64_t*)row; - time_t t = v/1000; - struct tm vtm = {0}; - localtime_r(&t, &vtm); - ts.year = (SQLSMALLINT)(vtm.tm_year + 1900); - ts.month = (SQLUSMALLINT)(vtm.tm_mon + 1); - ts.day = (SQLUSMALLINT)(vtm.tm_mday); - ts.hour = (SQLUSMALLINT)(vtm.tm_hour); - ts.minute = (SQLUSMALLINT)(vtm.tm_min); - ts.second = (SQLUSMALLINT)(vtm.tm_sec); - ts.fraction = (SQLUINTEGER)(v%1000 * 1000000); - switch (target.ct) { - case SQL_C_SBIGINT: { - *(int64_t*)TargetValue = v; - return SQL_SUCCESS; - } break; - case SQL_C_CHAR: { - tsdb_conv_t *utf8_to_client = tsdb_conn_utf8_to_client(conn); - size_t len = (size_t)BufferLength; - TSDB_CONV_CODE code = tsdb_conv_write_timestamp(utf8_to_client, ts, (char*)TargetValue, &len); - if (StrLen_or_Ind) *StrLen_or_Ind = (SQLLEN)len; - CHK_CONV(0, code); - } break; - case SQL_C_TYPE_TIMESTAMP: { - *(SQL_TIMESTAMP_STRUCT*)TargetValue = ts; - return SQL_SUCCESS; - } break; - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, - "no convertion from [%s] to [%s[%d][0x%x]] for col [%d]", - taos_data_type(field->type), sql_c_type(target.ct), target.ct, target.ct, ColumnNumber); - return SQL_ERROR; - } - } - } break; - case TSDB_DATA_TYPE_BINARY: { - size_t field_bytes = (size_t)field->bytes; - field_bytes -= VARSTR_HEADER_SIZE; - switch (target.ct) { - case SQL_C_CHAR: { - // taos cares nothing about what would be stored in 'binary' as most sql implementations do - // but the client requires to fetch it as a SQL_C_CHAR - // thus, we first try to decode binary to client charset - // if failed, we then do hex-serialization - - tsdb_conv_t *server_to_client = tsdb_conn_server_to_client(conn); - size_t slen = strnlen((const char*)row, field_bytes); - size_t len = (size_t)BufferLength; - TSDB_CONV_CODE code = tsdb_conv_write(server_to_client, - (const char*)row, &slen, - (char*)TargetValue, &len); - if (code==TSDB_CONV_OK) { - if (StrLen_or_Ind) *StrLen_or_Ind = (SQLLEN)((size_t)BufferLength - len); - CHK_CONV(0, code); - // code never reached here - } - - // todo: hex-serialization - const char *bad = ""; - int n = snprintf((char*)TargetValue, (size_t)BufferLength, "%s", bad); - // what if n < 0 ? - if (StrLen_or_Ind) *StrLen_or_Ind = n; - CHK_CONV(0, n>=BufferLength ? TSDB_CONV_TRUNC : TSDB_CONV_OK); - } break; - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, - "no convertion from [%s] to [%s[%d][0x%x]] for col [%d]", - taos_data_type(field->type), sql_c_type(target.ct), target.ct, target.ct, ColumnNumber); - return SQL_ERROR; - } - } - } break; - case TSDB_DATA_TYPE_NCHAR: { - size_t field_bytes = (size_t)field->bytes; - field_bytes -= VARSTR_HEADER_SIZE; - switch (target.ct) { - case SQL_C_CHAR: { - tsdb_conv_t *server_to_client = tsdb_conn_server_to_client(conn); - size_t slen = strnlen((const char*)row, field_bytes); - size_t len = (size_t)BufferLength; - TSDB_CONV_CODE code = tsdb_conv_write(server_to_client, - (const char*)row, &slen, - (char*)TargetValue, &len); - if (StrLen_or_Ind) *StrLen_or_Ind = (SQLLEN)((size_t)BufferLength - len); - CHK_CONV(0, code); - } break; - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, - "no convertion from [%s] to [%s[%d][0x%x]] for col [%d]", - taos_data_type(field->type), sql_c_type(target.ct), target.ct, target.ct, ColumnNumber); - return SQL_ERROR; - } - } - } break; - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for col [%d]", - taos_data_type(field->type), field->type, field->type, - sql_c_type(target.ct), target.ct, target.ct, ColumnNumber); - return SQL_ERROR; - } break; - } -} - -SQLRETURN SQL_API SQLGetData(SQLHSTMT StatementHandle, - SQLUSMALLINT ColumnNumber, SQLSMALLINT TargetType, - SQLPOINTER TargetValue, SQLLEN BufferLength, - SQLLEN *StrLen_or_Ind) -{ - SQLRETURN r; - r = doSQLGetData(StatementHandle, ColumnNumber, TargetType, - TargetValue, BufferLength, - StrLen_or_Ind); - return r; -} - -static SQLRETURN doSQLFetch(SQLHSTMT StatementHandle) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - if (!sql->rs) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NO_RESULT, ""); - return SQL_ERROR; - } - - sql->row = taos_fetch_row(sql->rs); - return sql->row ? SQL_SUCCESS : SQL_NO_DATA; -} - -SQLRETURN SQL_API SQLFetch(SQLHSTMT StatementHandle) -{ - SQLRETURN r; - r = doSQLFetch(StatementHandle); - return r; -} - -static SQLRETURN doSQLPrepare(SQLHSTMT StatementHandle, - SQLCHAR *StatementText, SQLINTEGER TextLength) -{ - stack_buffer_t buffer; buffer.next = 0; - - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - conn_t *conn = sql->conn; - - NORM_STR_LENGTH(sql, StatementText, TextLength); - - if (sql->rs) { - taos_free_result(sql->rs); - sql->rs = NULL; - sql->row = NULL; - } - - if (sql->stmt) { - taos_stmt_close(sql->stmt); - sql->stmt = NULL; - } - - if (sql->params) { - free(sql->params); - sql->params = NULL; - } - sql->n_params = 0; - sql->is_insert = 0; - - do { - sql->stmt = taos_stmt_init(sql->conn->taos); - if (!sql->stmt) { - SET_ERROR(sql, "HY001", terrno, "failed to initialize TAOS statement internally"); - break; - } - - tsdb_conv_t *client_to_server = tsdb_conn_client_to_server(conn); - const char *stxt = NULL; - int ok = 0; - do { - tsdb_conv(client_to_server, &buffer, (const char*)StatementText, (size_t)TextLength, &stxt, NULL); - if ((!stxt)) { - SET_ERROR(sql, "HY001", TSDB_CODE_ODBC_OOM, ""); - break; - } - - int r = taos_stmt_prepare(sql->stmt, stxt, (unsigned long)strlen(stxt)); - if (r) { - SET_ERROR(sql, "HY000", r, "failed to prepare a TAOS statement"); - break; - } - sql->is_prepared = 1; - - int is_insert = 0; - r = taos_stmt_is_insert(sql->stmt, &is_insert); - if (r) { - SET_ERROR(sql, "HY000", r, "failed to determine if a prepared-statement is of insert"); - break; - } - sql->is_insert = is_insert ? 1 : 0; - - int params = 0; - r = taos_stmt_num_params(sql->stmt, ¶ms); - if (r) { - SET_ERROR(sql, "HY000", terrno, "fetch num of statement params failed"); - break; - } - DASSERT(params>=0); - - if (params>0) { - param_bind_t *ar = (param_bind_t*)calloc(1, ((size_t)params) * sizeof(*ar)); - if (!ar) { - SET_ERROR(sql, "HY001", TSDB_CODE_ODBC_OOM, ""); - break; - } - sql->params = ar; - } - - sql->n_params = params; - - ok = 1; - } while (0); - - tsdb_conv_free(client_to_server, stxt, &buffer, (const char*)StatementText); - - if (!ok) { - taos_stmt_close(sql->stmt); - sql->stmt = NULL; - sql->is_prepared = 0; - sql->is_insert = 0; - sql->is_executed = 0; - } - } while (0); - - return sql->stmt ? SQL_SUCCESS : SQL_ERROR; -} - -SQLRETURN SQL_API SQLPrepare(SQLHSTMT StatementHandle, - SQLCHAR *StatementText, SQLINTEGER TextLength) -{ - SQLRETURN r; - r = doSQLPrepare(StatementHandle, StatementText, TextLength); - return r; -} - -static const int yes = 1; -static const int no = 0; - -static SQLRETURN do_bind_param_value(sql_t *sql, int idx_row, int idx, param_bind_t *param, TAOS_BIND *bind) -{ - if (!param->valid) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NOT_SUPPORT, "parameter [@%d] not bound yet", idx+1); - return SQL_ERROR; - } - if (param->ParameterValue==NULL) { - SET_ERROR(sql, "HY009", TSDB_CODE_ODBC_BAD_ARG, "ParameterValue [@%p] not allowed", param->ParameterValue); - return SQL_ERROR; - } - if (param->StrLen_or_Ind==NULL) { - SET_ERROR(sql, "HY009", TSDB_CODE_ODBC_BAD_ARG, "StrLen_or_Ind [@%p] not allowed", param->StrLen_or_Ind); - return SQL_ERROR; - } - - conn_t *conn = sql->conn; - - unsigned char *paramValue = param->ParameterValue; - SQLSMALLINT valueType = param->ValueType; - SQLLEN *soi = param->StrLen_or_Ind; - - size_t offset = ((size_t)idx_row) * sql->rowlen + sql->ptr_offset; - - paramValue += offset; - soi = (SQLLEN*)((char*)soi + offset); - - - if (*soi == SQL_NULL_DATA) { - bind->is_null = (int*)&yes; - return SQL_SUCCESS; - } - bind->is_null = (int*)&no; - int tsdb_type = 0; // taos internal data tsdb_type to be bound to - int tsdb_bytes = 0; // we don't rely much on 'tsdb_bytes' here, we delay until taos to check it internally - if (sql->is_insert) { - int r = taos_stmt_get_param(sql->stmt, idx, &tsdb_type, &tsdb_bytes); - if (r) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_OUT_OF_RANGE, "parameter [@%d] not valid", idx+1); - return SQL_ERROR; - } - } else { - // we don't have correspondent data type from taos api - // we have to give a good guess here - switch (valueType) { - case SQL_C_BIT: { - tsdb_type = TSDB_DATA_TYPE_BOOL; - } break; - case SQL_C_STINYINT: - case SQL_C_TINYINT: { - tsdb_type = TSDB_DATA_TYPE_TINYINT; - } break; - case SQL_C_SSHORT: - case SQL_C_SHORT: { - tsdb_type = TSDB_DATA_TYPE_SMALLINT; - } break; - case SQL_C_SLONG: - case SQL_C_LONG: { - tsdb_type = TSDB_DATA_TYPE_INT; - } break; - case SQL_C_SBIGINT: { - tsdb_type = TSDB_DATA_TYPE_BIGINT; - } break; - case SQL_C_FLOAT: { - tsdb_type = TSDB_DATA_TYPE_FLOAT; - } break; - case SQL_C_DOUBLE: { - tsdb_type = TSDB_DATA_TYPE_DOUBLE; - } break; - case SQL_C_TIMESTAMP: { - tsdb_type = TSDB_DATA_TYPE_TIMESTAMP; - } break; - case SQL_C_CHAR: { - tsdb_type = TSDB_DATA_TYPE_BINARY; - tsdb_bytes = SQL_NTS; - } break; - case SQL_C_WCHAR: { - tsdb_type = TSDB_DATA_TYPE_NCHAR; - tsdb_bytes = SQL_NTS; - } break; - case SQL_C_USHORT: - case SQL_C_ULONG: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - idx+1); - return SQL_ERROR; - } break; - } - } - - // ref: https://docs.microsoft.com/en-us/sql/odbc/reference/appendixes/converting-data-from-c-to-sql-data-types?view=sql-server-ver15 - switch (tsdb_type) { - case TSDB_DATA_TYPE_BOOL: { - bind->buffer_type = tsdb_type; - bind->buffer_length = sizeof(bind->u.b); - bind->buffer = &bind->u.b; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_BIT: { - CHK_CONV(1, tsdb_int64_to_bit(*(int8_t*)paramValue, &bind->u.b)); - } break; - case SQL_C_TINYINT: - case SQL_C_STINYINT: { - CHK_CONV(1, tsdb_int64_to_bit(*(int8_t*)paramValue, &bind->u.b)); - } break; - case SQL_C_SHORT: - case SQL_C_SSHORT: { - CHK_CONV(1, tsdb_int64_to_bit(*(int16_t*)paramValue, &bind->u.b)); - } break; - case SQL_C_LONG: - case SQL_C_SLONG: { - CHK_CONV(1, tsdb_int64_to_bit(*(int32_t*)paramValue, &bind->u.b)); - } break; - case SQL_C_SBIGINT: { - CHK_CONV(1, tsdb_int64_to_bit(*(int64_t*)paramValue, &bind->u.b)); - } break; - case SQL_C_FLOAT: { - CHK_CONV(1, tsdb_double_to_bit(*(float*)paramValue, &bind->u.b)); - } break; - case SQL_C_DOUBLE: { - CHK_CONV(1, tsdb_double_to_bit(*(double*)paramValue, &bind->u.b)); - } break; - case SQL_C_CHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_utf8 = tsdb_conn_client_to_utf8(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - CHK_CONV(1, tsdb_conv_chars_to_bit(client_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.b)); - } break; - case SQL_C_WCHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_utf8 = tsdb_conn_utf16_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_bit(utf16_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.b)); - } break; - case SQL_C_USHORT: - case SQL_C_ULONG: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_TINYINT: { - bind->buffer_type = tsdb_type; - bind->buffer_length = sizeof(bind->u.v1); - bind->buffer = &bind->u.v1; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_BIT: { - CHK_CONV(1, tsdb_int64_to_tinyint(*(int8_t*)paramValue, &bind->u.v1)); - } break; - case SQL_C_STINYINT: - case SQL_C_TINYINT: { - CHK_CONV(1, tsdb_int64_to_tinyint(*(int8_t*)paramValue, &bind->u.v1)); - } break; - case SQL_C_SSHORT: - case SQL_C_SHORT: { - CHK_CONV(1, tsdb_int64_to_tinyint(*(int16_t*)paramValue, &bind->u.v1)); - } break; - case SQL_C_SLONG: - case SQL_C_LONG: { - CHK_CONV(1, tsdb_int64_to_tinyint(*(int32_t*)paramValue, &bind->u.v1)); - } break; - case SQL_C_SBIGINT: { - CHK_CONV(1, tsdb_int64_to_tinyint(*(int64_t*)paramValue, &bind->u.v1)); - } break; - case SQL_C_CHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_utf8 = tsdb_conn_client_to_utf8(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - CHK_CONV(1, tsdb_conv_chars_to_tinyint(client_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v1)); - // CHK_CONV(1, tsdb_chars_to_tinyint((const char *)paramValue, (size_t)*soi, &bind->u.v1)); - } break; - case SQL_C_WCHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_utf8 = tsdb_conn_utf16_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_tinyint(utf16_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v1)); - } break; - case SQL_C_USHORT: - case SQL_C_ULONG: - case SQL_C_FLOAT: - case SQL_C_DOUBLE: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_SMALLINT: { - bind->buffer_type = tsdb_type; - bind->buffer_length = sizeof(bind->u.v2); - bind->buffer = &bind->u.v2; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_BIT: { - CHK_CONV(1, tsdb_int64_to_smallint(*(int8_t*)paramValue, &bind->u.v2)); - } break; - case SQL_C_STINYINT: - case SQL_C_TINYINT: { - CHK_CONV(1, tsdb_int64_to_smallint(*(int8_t*)paramValue, &bind->u.v2)); - } break; - case SQL_C_SSHORT: - case SQL_C_SHORT: { - CHK_CONV(1, tsdb_int64_to_smallint(*(int16_t*)paramValue, &bind->u.v2)); - } break; - case SQL_C_SLONG: - case SQL_C_LONG: { - CHK_CONV(1, tsdb_int64_to_smallint(*(int32_t*)paramValue, &bind->u.v2)); - } break; - case SQL_C_SBIGINT: { - CHK_CONV(1, tsdb_int64_to_smallint(*(int64_t*)paramValue, &bind->u.v2)); - } break; - case SQL_C_CHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_utf8 = tsdb_conn_client_to_utf8(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - CHK_CONV(1, tsdb_conv_chars_to_smallint(client_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v2)); - // CHK_CONV(1, tsdb_chars_to_smallint((const char*)paramValue, (size_t)*soi, &bind->u.v2)); - } break; - case SQL_C_WCHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_utf8 = tsdb_conn_utf16_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_smallint(utf16_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v2)); - } break; - case SQL_C_USHORT: - case SQL_C_ULONG: - case SQL_C_FLOAT: - case SQL_C_DOUBLE: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_INT: { - bind->buffer_type = tsdb_type; - bind->buffer_length = sizeof(bind->u.v4); - bind->buffer = &bind->u.v4; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_BIT: { - CHK_CONV(1, tsdb_int64_to_int(*(int8_t*)paramValue, &bind->u.v4)); - } break; - case SQL_C_STINYINT: - case SQL_C_TINYINT: { - CHK_CONV(1, tsdb_int64_to_int(*(int8_t*)paramValue, &bind->u.v4)); - } break; - case SQL_C_SSHORT: - case SQL_C_SHORT: { - CHK_CONV(1, tsdb_int64_to_int(*(int16_t*)paramValue, &bind->u.v4)); - } break; - case SQL_C_SLONG: - case SQL_C_LONG: { - CHK_CONV(1, tsdb_int64_to_int(*(int32_t*)paramValue, &bind->u.v4)); - } break; - case SQL_C_SBIGINT: { - CHK_CONV(1, tsdb_int64_to_int(*(int64_t*)paramValue, &bind->u.v4)); - } break; - case SQL_C_CHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_utf8 = tsdb_conn_client_to_utf8(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - CHK_CONV(1, tsdb_conv_chars_to_int(client_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v4)); - // CHK_CONV(1, tsdb_chars_to_int((const char*)paramValue, (size_t)*soi, &bind->u.v4)); - } break; - case SQL_C_WCHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_utf8 = tsdb_conn_utf16_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_int(utf16_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v4)); - } break; - case SQL_C_USHORT: - case SQL_C_ULONG: - case SQL_C_FLOAT: - case SQL_C_DOUBLE: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_BIGINT: { - bind->buffer_type = tsdb_type; - bind->buffer_length = sizeof(bind->u.v8); - bind->buffer = &bind->u.v8; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_BIT: { - CHK_CONV(1, tsdb_int64_to_bigint(*(int8_t*)paramValue, &bind->u.v8)); - } break; - case SQL_C_STINYINT: - case SQL_C_TINYINT: { - CHK_CONV(1, tsdb_int64_to_bigint(*(int8_t*)paramValue, &bind->u.v8)); - } break; - case SQL_C_SSHORT: - case SQL_C_SHORT: { - CHK_CONV(1, tsdb_int64_to_bigint(*(int16_t*)paramValue, &bind->u.v8)); - } break; - case SQL_C_SLONG: - case SQL_C_LONG: { - CHK_CONV(1, tsdb_int64_to_bigint(*(int32_t*)paramValue, &bind->u.v8)); - } break; - case SQL_C_SBIGINT: { - CHK_CONV(1, tsdb_int64_to_bigint(*(int64_t*)paramValue, &bind->u.v8)); - } break; - case SQL_C_CHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_utf8 = tsdb_conn_client_to_utf8(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - CHK_CONV(1, tsdb_conv_chars_to_bigint(client_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v8)); - // CHK_CONV(1, tsdb_chars_to_bigint((const char*)paramValue, (size_t)*soi, &bind->u.v8)); - } break; - case SQL_C_WCHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_utf8 = tsdb_conn_utf16_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_bigint(utf16_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v8)); - } break; - case SQL_C_USHORT: - case SQL_C_ULONG: - case SQL_C_FLOAT: - case SQL_C_DOUBLE: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_FLOAT: { - bind->buffer_type = tsdb_type; - bind->buffer_length = sizeof(bind->u.f4); - bind->buffer = &bind->u.f4; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_BIT: { - CHK_CONV(1, tsdb_int64_to_float(*(int8_t*)paramValue, &bind->u.f4)); - } break; - case SQL_C_STINYINT: - case SQL_C_TINYINT: { - CHK_CONV(1, tsdb_int64_to_float(*(int8_t*)paramValue, &bind->u.f4)); - } break; - case SQL_C_SSHORT: - case SQL_C_SHORT: { - CHK_CONV(1, tsdb_int64_to_float(*(int16_t*)paramValue, &bind->u.f4)); - } break; - case SQL_C_SLONG: - case SQL_C_LONG: { - CHK_CONV(1, tsdb_int64_to_float(*(int32_t*)paramValue, &bind->u.f4)); - } break; - case SQL_C_SBIGINT: { - CHK_CONV(1, tsdb_int64_to_float(*(int64_t*)paramValue, &bind->u.f4)); - } break; - case SQL_C_FLOAT: { - bind->u.f4 = *(float*)paramValue; - } break; - case SQL_C_DOUBLE: { - bind->u.f4 = (float)*(double*)paramValue; - } break; - case SQL_C_CHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_utf8 = tsdb_conn_client_to_utf8(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - CHK_CONV(1, tsdb_conv_chars_to_float(client_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.f4)); - } break; - case SQL_C_WCHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_utf8 = tsdb_conn_utf16_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_float(utf16_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.f4)); - } break; - case SQL_C_USHORT: - case SQL_C_ULONG: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_DOUBLE: { - bind->buffer_type = tsdb_type; - bind->buffer_length = sizeof(bind->u.f8); - bind->buffer = &bind->u.f8; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_BIT: { - CHK_CONV(1, tsdb_int64_to_double(*(int8_t*)paramValue, &bind->u.f8)); - } break; - case SQL_C_STINYINT: - case SQL_C_TINYINT: { - CHK_CONV(1, tsdb_int64_to_double(*(int8_t*)paramValue, &bind->u.f8)); - } break; - case SQL_C_SSHORT: - case SQL_C_SHORT: { - CHK_CONV(1, tsdb_int64_to_double(*(int16_t*)paramValue, &bind->u.f8)); - } break; - case SQL_C_SLONG: - case SQL_C_LONG: { - CHK_CONV(1, tsdb_int64_to_double(*(int32_t*)paramValue, &bind->u.f8)); - } break; - case SQL_C_SBIGINT: { - CHK_CONV(1, tsdb_int64_to_double(*(int64_t*)paramValue, &bind->u.f8)); - } break; - case SQL_C_FLOAT: { - bind->u.f8 = *(float*)paramValue; - } break; - case SQL_C_DOUBLE: { - bind->u.f8 = *(double*)paramValue; - } break; - case SQL_C_CHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_utf8 = tsdb_conn_client_to_utf8(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - CHK_CONV(1, tsdb_conv_chars_to_double(client_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.f8)); - // CHK_CONV(1, tsdb_chars_to_double((const char*)paramValue, (size_t)*soi, &bind->u.f8)); - } break; - case SQL_C_WCHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_utf8 = tsdb_conn_utf16_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_double(utf16_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.f8)); - } break; - case SQL_C_USHORT: - case SQL_C_ULONG: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_TIMESTAMP: { - bind->buffer_type = tsdb_type; - bind->buffer_length = sizeof(bind->u.v8); - bind->buffer = &bind->u.v8; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_CHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *client_to_utf8 = tsdb_conn_client_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_timestamp_ts(client_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v8)); - } break; - case SQL_C_WCHAR: { - stack_buffer_t buffer; buffer.next = 0; - tsdb_conv_t *utf16_to_utf8 = tsdb_conn_utf16_to_utf8(conn); - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - CHK_CONV(1, tsdb_conv_chars_to_timestamp_ts(utf16_to_utf8, &buffer, (const char *)paramValue, slen, &bind->u.v8)); - } break; - case SQL_C_SBIGINT: { - int64_t t = *(int64_t*)paramValue; - bind->u.v8 = t; - } break; - case SQL_C_TYPE_TIMESTAMP: { - SQL_TIMESTAMP_STRUCT ts = *(SQL_TIMESTAMP_STRUCT*)paramValue; - struct tm vtm = {0}; - vtm.tm_year = ts.year - 1900; - vtm.tm_mon = ts.month - 1; - vtm.tm_mday = ts.day; - vtm.tm_hour = ts.hour; - vtm.tm_min = ts.minute; - vtm.tm_sec = ts.second; - int64_t t = (int64_t) mktime(&vtm); - if (t==-1) { - CHK_CONV(1, TSDB_CONV_NOT_VALID_TS); - // code never reached here - } - bind->u.ts = t * 1000 + ts.fraction / 1000000; - } break; - case SQL_C_SHORT: - case SQL_C_SSHORT: - case SQL_C_USHORT: - case SQL_C_LONG: - case SQL_C_SLONG: - case SQL_C_ULONG: - case SQL_C_FLOAT: - case SQL_C_DOUBLE: - case SQL_C_BIT: - case SQL_C_TINYINT: - case SQL_C_STINYINT: - case SQL_C_UTINYINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_BINARY: { - bind->buffer_type = tsdb_type; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_WCHAR: { - // taos cares nothing about what would be stored in 'binary' as most sql implementations do - // thus, we just copy it as is - // it's caller's responsibility to maintain data-consistency - // if he/she is going to use 'binary' to store characters - // taos might extend it's sql syntax to let user specify - // what charset is to be used for specific 'binary' field when - // table is to be created - // in such way, 'binary' would be 'internationalized' - // but actually speaking, normally, 'char' field is a better - // one for this purpose - size_t slen = (size_t)*soi; - DASSERT(slen != SQL_NTS); - bind->u.bin = (unsigned char*)malloc(slen + 1); // add null-terminator, just for case of use - if (!bind->u.bin) { - CHK_CONV(1, TSDB_CONV_OOM); - // code never reached here - } - memcpy(bind->u.bin, paramValue, slen); - bind->buffer_length = slen; - bind->buffer = bind->u.bin; - CHK_CONV(1, TSDB_CONV_OK); - - // tsdb_conv_t *utf16_to_server = tsdb_conn_utf16_to_server(conn); - // size_t slen = (size_t)*soi; - // DASSERT(slen != SQL_NTS); - // const char *buf = NULL; - // size_t blen = 0; - // TSDB_CONV_CODE code = tsdb_conv(utf16_to_server, NULL, (const char *)paramValue, slen, &buf, &blen); - // if (code==TSDB_CONV_OK) { - // if (buf!=(const char*)paramValue) { - // bind->allocated = 1; - // } - // bind->u.bin = (unsigned char*)buf; - // bind->buffer_length = blen; - // bind->buffer = bind->u.bin; - // } - // CHK_CONV(1, code); - } break; - case SQL_C_CHAR: { - // taos cares nothing about what would be stored in 'binary' as most sql implementations do - // thus, we just copy it as is - // it's caller's responsibility to maintain data-consistency - // if he/she is going to use 'binary' to store characters - // taos might extend it's sql syntax to let user specify - // what charset is to be used for specific 'binary' field when - // table is to be created - // in such way, 'binary' would be 'internationalized' - // but actually speaking, normally, 'char' field is a better - // one for this purpose - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - // we can not use strndup, because ODBC client might pass in a buffer without null-terminated - bind->u.bin = (unsigned char*)malloc(slen + 1); // add null-terminator, just for case of use - if (!bind->u.bin) { - CHK_CONV(1, TSDB_CONV_OOM); - // code never reached here - } - memcpy(bind->u.bin, paramValue, slen); - bind->buffer_length = slen; - bind->buffer = bind->u.bin; - CHK_CONV(1, TSDB_CONV_OK); - // code never reached here - - // tsdb_conv_t *client_to_server = tsdb_conn_client_to_server(conn); - // size_t slen = (size_t)*soi; - // if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - // const char *buf = NULL; - // size_t blen = 0; - // TSDB_CONV_CODE code = tsdb_conv(client_to_server, NULL, (const char *)paramValue, slen, &buf, &blen); - // if (code==TSDB_CONV_OK) { - // if (buf!=(const char*)paramValue) { - // bind->allocated = 1; - // } - // bind->u.bin = (unsigned char*)buf; - // bind->buffer_length = blen; - // bind->buffer = bind->u.bin; - // } - // CHK_CONV(1, code); - } break; - case SQL_C_SHORT: - case SQL_C_SSHORT: - case SQL_C_USHORT: - case SQL_C_LONG: - case SQL_C_SLONG: - case SQL_C_ULONG: - case SQL_C_FLOAT: - case SQL_C_DOUBLE: - case SQL_C_BIT: - case SQL_C_TINYINT: - case SQL_C_STINYINT: - case SQL_C_UTINYINT: - case SQL_C_SBIGINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: // we don't provide auto-converstion - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - case TSDB_DATA_TYPE_NCHAR: { - bind->buffer_type = tsdb_type; - bind->length = &bind->buffer_length; - switch (valueType) { - case SQL_C_WCHAR: { - tsdb_conv_t *utf16_to_server = tsdb_conn_utf16_to_server(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - const char *buf = NULL; - size_t blen = 0; - TSDB_CONV_CODE code = tsdb_conv(utf16_to_server, NULL, (const char *)paramValue, slen, &buf, &blen); - if (code==TSDB_CONV_OK) { - if (buf!=(const char*)paramValue) { - bind->allocated = 1; - } - bind->u.nchar = (char*)buf; - bind->buffer_length = blen; - bind->buffer = bind->u.nchar; - } - CHK_CONV(1, code); - } break; - case SQL_C_CHAR: { - tsdb_conv_t *client_to_server = tsdb_conn_client_to_server(conn); - size_t slen = (size_t)*soi; - if (slen==SQL_NTS) slen = strlen((const char*)paramValue); - const char *buf = NULL; - size_t blen = 0; - TSDB_CONV_CODE code = tsdb_conv(client_to_server, NULL, (const char *)paramValue, slen, &buf, &blen); - if (code==TSDB_CONV_OK) { - if (buf!=(const char*)paramValue) { - bind->allocated = 1; - } - bind->u.bin = (unsigned char*)buf; - bind->buffer_length = blen; - bind->buffer = bind->u.bin; - } - CHK_CONV(1, code); - } break; - case SQL_C_SHORT: - case SQL_C_SSHORT: - case SQL_C_USHORT: - case SQL_C_LONG: - case SQL_C_SLONG: - case SQL_C_ULONG: - case SQL_C_FLOAT: - case SQL_C_DOUBLE: - case SQL_C_BIT: - case SQL_C_TINYINT: - case SQL_C_STINYINT: - case SQL_C_UTINYINT: - case SQL_C_SBIGINT: - case SQL_C_UBIGINT: - case SQL_C_BINARY: - case SQL_C_DATE: - case SQL_C_TIME: - case SQL_C_TIMESTAMP: - case SQL_C_TYPE_DATE: - case SQL_C_TYPE_TIME: - case SQL_C_TYPE_TIMESTAMP: // we don't provide auto-converstion - case SQL_C_NUMERIC: - case SQL_C_GUID: - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - } break; - default: { - SET_ERROR(sql, "HYC00", TSDB_CODE_ODBC_OUT_OF_RANGE, - "no convertion from [%s[%d/0x%x]] to [%s[%d/0x%x]] for parameter [%d]", - sql_c_type(valueType), valueType, valueType, - taos_data_type(tsdb_type), tsdb_type, tsdb_type, idx+1); - return SQL_ERROR; - } break; - } - return SQL_SUCCESS; -} - -static SQLRETURN do_bind_batch(sql_t *sql, int idx_row, TAOS_BIND *binds) -{ - for (int j=0; jn_params; ++j) { - SQLRETURN r = do_bind_param_value(sql, idx_row, j, sql->params+j, binds+j); - if (r==SQL_SUCCESS) continue; - return r; - } - if (sql->n_params > 0) { - int tr = 0; - PROFILE(tr = taos_stmt_bind_param(sql->stmt, binds)); - if (tr) { - SET_ERROR(sql, "HY000", tr, "failed to bind parameters[%d in total]", sql->n_params); - return SQL_ERROR; - } - - if (sql->is_insert) { - int r = 0; - PROFILE(r = taos_stmt_add_batch(sql->stmt)); - if (r) { - SET_ERROR(sql, "HY000", r, "failed to add batch"); - return SQL_ERROR; - } - } - } - return SQL_SUCCESS; -} - -static SQLRETURN do_execute(sql_t *sql) -{ - int tr = TSDB_CODE_SUCCESS; - if (sql->n_rows==0) sql->n_rows = 1; - for (int i=0; in_rows; ++i) { - TAOS_BIND *binds = NULL; - if (sql->n_params>0) { - binds = (TAOS_BIND*)calloc((size_t)sql->n_params, sizeof(*binds)); - if (!binds) { - SET_ERROR(sql, "HY001", TSDB_CODE_ODBC_OOM, ""); - return SQL_ERROR; - } - } - - SQLRETURN r = do_bind_batch(sql, i, binds); - - if (binds) { - for (int i = 0; in_params; ++i) { - TAOS_BIND *bind = binds + i; - if (bind->allocated) { - free(bind->u.nchar); - bind->u.nchar = NULL; - } - } - free(binds); - } - - if (r) return r; - } - - PROFILE(tr = taos_stmt_execute(sql->stmt)); - if (tr) { - SET_ERROR(sql, "HY000", tr, "failed to execute statement"); - return SQL_ERROR; - } - - sql->is_executed = 1; - // if (sql->is_insert) return SQL_SUCCESS; - - SQLRETURN r = SQL_SUCCESS; - PROFILE(sql->rs = taos_stmt_use_result(sql->stmt)); - CHK_RS(r, sql, "failed to use result"); - - return r; -} - -static SQLRETURN doSQLExecute(SQLHSTMT StatementHandle) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_ERROR; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - if (!sql->stmt) { - SET_ERROR(sql, "HY010", TSDB_CODE_ODBC_STATEMENT_NOT_READY, ""); - return SQL_ERROR; - } - - if (sql->rs) { - taos_free_result(sql->rs); - sql->rs = NULL; - sql->row = NULL; - } - - SQLRETURN r = do_execute(sql); - - return r; -} - -SQLRETURN SQL_API SQLExecute(SQLHSTMT StatementHandle) -{ - SQLRETURN r; - PROFILE(r = doSQLExecute(StatementHandle)); - return r; -} - -static SQLRETURN doSQLGetDiagField(SQLSMALLINT HandleType, SQLHANDLE Handle, - SQLSMALLINT RecNumber, SQLSMALLINT DiagIdentifier, - SQLPOINTER DiagInfo, SQLSMALLINT BufferLength, - SQLSMALLINT *StringLength) -{ - switch (DiagIdentifier) { - case SQL_DIAG_CLASS_ORIGIN: { - *StringLength = 0; - } break; - } - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLGetDiagField(SQLSMALLINT HandleType, SQLHANDLE Handle, - SQLSMALLINT RecNumber, SQLSMALLINT DiagIdentifier, - SQLPOINTER DiagInfo, SQLSMALLINT BufferLength, - SQLSMALLINT *StringLength) -{ - SQLRETURN r; - r = doSQLGetDiagField(HandleType, Handle, - RecNumber, DiagIdentifier, - DiagInfo, BufferLength, - StringLength); - return r; -} - -static SQLRETURN doSQLGetDiagRec(SQLSMALLINT HandleType, SQLHANDLE Handle, - SQLSMALLINT RecNumber, SQLCHAR *Sqlstate, - SQLINTEGER *NativeError, SQLCHAR *MessageText, - SQLSMALLINT BufferLength, SQLSMALLINT *TextLength) -{ - if (RecNumber>1) return SQL_NO_DATA; - - switch (HandleType) { - case SQL_HANDLE_ENV: { - env_t *env = (env_t*)Handle; - if (!env) break; - FILL_ERROR(env); - return SQL_SUCCESS; - } break; - case SQL_HANDLE_DBC: { - conn_t *conn = (conn_t*)Handle; - if (!conn) break; - FILL_ERROR(conn); - return SQL_SUCCESS; - } break; - case SQL_HANDLE_STMT: { - sql_t *sql = (sql_t*)Handle; - if (!sql) break; - FILL_ERROR(sql); - return SQL_SUCCESS; - } break; - default: { - } break; - } - - // how to return error? - return SQL_ERROR; -} - -SQLRETURN SQL_API SQLGetDiagRec(SQLSMALLINT HandleType, SQLHANDLE Handle, - SQLSMALLINT RecNumber, SQLCHAR *Sqlstate, - SQLINTEGER *NativeError, SQLCHAR *MessageText, - SQLSMALLINT BufferLength, SQLSMALLINT *TextLength) -{ - SQLRETURN r; - r = doSQLGetDiagRec(HandleType, Handle, - RecNumber, Sqlstate, - NativeError, MessageText, - BufferLength, TextLength); - return r; -} - -static SQLRETURN doSQLBindParameter( - SQLHSTMT StatementHandle, - SQLUSMALLINT ParameterNumber, - SQLSMALLINT fParamType, - SQLSMALLINT ValueType, - SQLSMALLINT ParameterType, - SQLULEN LengthPrecision, - SQLSMALLINT ParameterScale, - SQLPOINTER ParameterValue, - SQLLEN cbValueMax, // ignore for now, since only SQL_PARAM_INPUT is supported now - SQLLEN *StrLen_or_Ind) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - if (!sql->stmt) { - SET_ERROR(sql, "HY010", TSDB_CODE_ODBC_STATEMENT_NOT_READY, ""); - return SQL_ERROR; - } - - if (ParameterNumber<=0 || ParameterNumber>sql->n_params) { - SET_ERROR(sql, "07009", TSDB_CODE_ODBC_BAD_ARG, - "parameter [@%d] invalid", ParameterNumber); - return SQL_ERROR; - } - - if (fParamType != SQL_PARAM_INPUT) { - SET_ERROR(sql, "HY105", TSDB_CODE_ODBC_NOT_SUPPORT, "non-input parameter [@%d] not supported yet", ParameterNumber); - return SQL_ERROR; - } - - if (ValueType == SQL_C_DEFAULT) { - SET_ERROR(sql, "HY003", TSDB_CODE_ODBC_NOT_SUPPORT, "default value for parameter [@%d] not supported yet", ParameterNumber); - return SQL_ERROR; - } - - if (!is_valid_sql_c_type(ValueType)) { - SET_ERROR(sql, "HY003", TSDB_CODE_ODBC_NOT_SUPPORT, - "SQL_C_TYPE [%s/%d/0x%x] for parameter [@%d] unknown", - sql_c_type(ValueType), ValueType, ValueType, ParameterNumber); - return SQL_ERROR; - } - - if (!is_valid_sql_sql_type(ParameterType)) { - SET_ERROR(sql, "HY004", TSDB_CODE_ODBC_NOT_SUPPORT, - "SQL_TYPE [%s/%d/0x%x] for parameter [@%d] unknown", - sql_c_type(ParameterType), ParameterType, ParameterType, ParameterNumber); - return SQL_ERROR; - } - - if (ParameterValue==NULL) { - SET_ERROR(sql, "HY009", TSDB_CODE_ODBC_BAD_ARG, "ParameterValue [@%p] not allowed", ParameterValue); - return SQL_ERROR; - } - - if (StrLen_or_Ind==NULL) { - SET_ERROR(sql, "HY009", TSDB_CODE_ODBC_BAD_ARG, "StrLen_or_Ind [@%p] not allowed", StrLen_or_Ind); - return SQL_ERROR; - } - - param_bind_t *pb = sql->params + ParameterNumber - 1; - - pb->ParameterNumber = ParameterNumber; - pb->ValueType = ValueType; - pb->ParameterType = ParameterType; - pb->LengthPrecision = LengthPrecision; - pb->ParameterScale = ParameterScale; - pb->ParameterValue = ParameterValue; - pb->StrLen_or_Ind = StrLen_or_Ind; - - pb->valid = 1; - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLBindParameter( - SQLHSTMT StatementHandle, - SQLUSMALLINT ParameterNumber, - SQLSMALLINT fParamType, - SQLSMALLINT ValueType, - SQLSMALLINT ParameterType, - SQLULEN LengthPrecision, - SQLSMALLINT ParameterScale, - SQLPOINTER ParameterValue, - SQLLEN cbValueMax, // ignore for now, since only SQL_PARAM_INPUT is supported now - SQLLEN *StrLen_or_Ind) -{ - SQLRETURN r; - r = doSQLBindParameter(StatementHandle, ParameterNumber, fParamType, ValueType, ParameterType, - LengthPrecision, ParameterScale, ParameterValue, cbValueMax, StrLen_or_Ind); - return r; -} - -static SQLRETURN doSQLDriverConnect( - SQLHDBC hdbc, - SQLHWND hwnd, - SQLCHAR *szConnStrIn, - SQLSMALLINT cbConnStrIn, - SQLCHAR *szConnStrOut, - SQLSMALLINT cbConnStrOutMax, - SQLSMALLINT *pcbConnStrOut, - SQLUSMALLINT fDriverCompletion) -{ - conn_t *conn = (conn_t*)hdbc; - if (!conn) return SQL_INVALID_HANDLE; - - if (conn->taos) { - SET_ERROR(conn, "08002", TSDB_CODE_ODBC_CONNECTION_BUSY, "connection still in use"); - return SQL_ERROR; - } - - if (fDriverCompletion!=SQL_DRIVER_NOPROMPT) { - SET_ERROR(conn, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, "option[%d] other than SQL_DRIVER_NOPROMPT not supported yet", fDriverCompletion); - return SQL_ERROR; - } - - NORM_STR_LENGTH(conn, szConnStrIn, cbConnStrIn); - - // DSN=; UID=; PWD= - - const char *connStr = SDUP(szConnStrIn, cbConnStrIn); - - conn_val_t val = {0}; - - do { - if (szConnStrIn && !connStr) { - SET_ERROR(conn, "HY001", TSDB_CODE_ODBC_OOM, ""); - break; - } - - int n = todbc_parse_conn_string((const char *)connStr, &val); - if (n) { - SET_ERROR(conn, "HY000", TSDB_CODE_ODBC_BAD_CONNSTR, "unrecognized connection string: [%s]", (const char*)szConnStrIn); - break; - } - char *ip = NULL; - int port = 0; - if (val.server) { - char *p = strchr(val.server, ':'); - if (p) { - ip = strndup(val.server, (size_t)(p-val.server)); - port = atoi(p+1); - } - } - - if ((val.cli_enc && strcmp(val.cli_enc, conn->client_enc)) || - (val.svr_enc && strcmp(val.svr_enc, conn->server_enc)) ) - { - tsdb_conn_close_convs(conn); - if (val.cli_enc) { - snprintf(conn->client_enc, sizeof(conn->client_enc), "%s", val.cli_enc); - } - if (val.svr_enc) { - snprintf(conn->server_enc, sizeof(conn->server_enc), "%s", val.svr_enc); - } - } - - // TODO: data-race - // TODO: shall receive ip/port from odbc.ini - // shall we support non-ansi uid/pwd/db etc? - conn->taos = taos_connect(ip ? ip : "localhost", val.uid, val.pwd, val.db, (uint16_t)port); - free(ip); ip = NULL; - if (!conn->taos) { - SET_ERROR(conn, "HY000", terrno, "failed to connect to data source"); - break; - } - - if (szConnStrOut) { - snprintf((char*)szConnStrOut, (size_t)cbConnStrOutMax, "%s", connStr); - } - if (pcbConnStrOut) { - *pcbConnStrOut = cbConnStrIn; - } - } while (0); - - conn_val_reset(&val); - - SFRE(connStr, szConnStrIn, cbConnStrIn); - - return conn->taos ? SQL_SUCCESS : SQL_ERROR; -} - -SQLRETURN SQL_API SQLDriverConnect( - SQLHDBC hdbc, - SQLHWND hwnd, - SQLCHAR *szConnStrIn, - SQLSMALLINT cbConnStrIn, - SQLCHAR *szConnStrOut, - SQLSMALLINT cbConnStrOutMax, - SQLSMALLINT *pcbConnStrOut, - SQLUSMALLINT fDriverCompletion) -{ - SQLRETURN r; - r = doSQLDriverConnect(hdbc, hwnd, szConnStrIn, cbConnStrIn, szConnStrOut, cbConnStrOutMax, pcbConnStrOut, fDriverCompletion); - return r; -} - -static SQLRETURN doSQLSetConnectAttr(SQLHDBC ConnectionHandle, - SQLINTEGER Attribute, SQLPOINTER Value, - SQLINTEGER StringLength) -{ - conn_t *conn = (conn_t*)ConnectionHandle; - if (!conn) return SQL_INVALID_HANDLE; - - if (Attribute != SQL_ATTR_AUTOCOMMIT) { - SET_ERROR(conn, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, "Attribute other than SQL_ATTR_AUTOCOMMIT not supported yet"); - return SQL_ERROR; - } - if (Value != (SQLPOINTER)SQL_AUTOCOMMIT_ON) { - SET_ERROR(conn, "HYC00", TSDB_CODE_ODBC_NOT_SUPPORT, "Attribute Value other than SQL_AUTOCOMMIT_ON not supported yet[%p]", Value); - return SQL_ERROR; - } - - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLSetConnectAttr(SQLHDBC ConnectionHandle, - SQLINTEGER Attribute, SQLPOINTER Value, - SQLINTEGER StringLength) -{ - SQLRETURN r; - r = doSQLSetConnectAttr(ConnectionHandle, Attribute, Value, StringLength); - return r; -} - -static SQLRETURN doSQLDescribeCol(SQLHSTMT StatementHandle, - SQLUSMALLINT ColumnNumber, SQLCHAR *ColumnName, - SQLSMALLINT BufferLength, SQLSMALLINT *NameLength, - SQLSMALLINT *DataType, SQLULEN *ColumnSize, - SQLSMALLINT *DecimalDigits, SQLSMALLINT *Nullable) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - if (!sql->rs) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NO_RESULT, ""); - return SQL_ERROR; - } - - int nfields = taos_field_count(sql->rs); - TAOS_FIELD *fields = taos_fetch_fields(sql->rs); - - if (ColumnNumber<=0 || ColumnNumber>nfields) { - SET_ERROR(sql, "07009", TSDB_CODE_ODBC_OUT_OF_RANGE, "invalid column number [%d]", ColumnNumber); - return SQL_ERROR; - } - if (BufferLength<0) { - SET_ERROR(sql, "HY090", TSDB_CODE_ODBC_BAD_ARG, ""); - return SQL_ERROR; - } - - TAOS_FIELD *field = fields + ColumnNumber - 1; - if (ColumnName) { - size_t n = sizeof(field->name); - if (n>BufferLength) n = (size_t)BufferLength; - strncpy((char*)ColumnName, field->name, n); - } - if (NameLength) { - *NameLength = (SQLSMALLINT)strnlen(field->name, sizeof(field->name)); - } - if (ColumnSize) *ColumnSize = (SQLULEN)field->bytes; - if (DecimalDigits) *DecimalDigits = 0; - - if (DataType) { - switch (field->type) { - case TSDB_DATA_TYPE_BOOL: { - *DataType = SQL_TINYINT; - } break; - - case TSDB_DATA_TYPE_TINYINT: { - *DataType = SQL_TINYINT; - } break; - - case TSDB_DATA_TYPE_SMALLINT: { - *DataType = SQL_SMALLINT; - } break; - - case TSDB_DATA_TYPE_INT: { - *DataType = SQL_INTEGER; - } break; - - case TSDB_DATA_TYPE_BIGINT: { - *DataType = SQL_BIGINT; - } break; - - case TSDB_DATA_TYPE_FLOAT: { - *DataType = SQL_FLOAT; - } break; - - case TSDB_DATA_TYPE_DOUBLE: { - *DataType = SQL_DOUBLE; - } break; - - case TSDB_DATA_TYPE_TIMESTAMP: { - *DataType = SQL_TIMESTAMP; - if (ColumnSize) *ColumnSize = sizeof(SQL_TIMESTAMP_STRUCT); - if (DecimalDigits) *DecimalDigits = 0; - } break; - - case TSDB_DATA_TYPE_NCHAR: { - *DataType = SQL_CHAR; // unicode ? - if (ColumnSize) *ColumnSize -= VARSTR_HEADER_SIZE; - } break; - - case TSDB_DATA_TYPE_BINARY: { - *DataType = SQL_CHAR; - if (ColumnSize) *ColumnSize -= VARSTR_HEADER_SIZE; - } break; - - default: - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NOT_SUPPORT, - "unknown [%s[%d/0x%x]]", taos_data_type(field->type), field->type, field->type); - return SQL_ERROR; - break; - } - } - if (Nullable) { - *Nullable = SQL_NULLABLE_UNKNOWN; - } - - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLDescribeCol(SQLHSTMT StatementHandle, - SQLUSMALLINT ColumnNumber, SQLCHAR *ColumnName, - SQLSMALLINT BufferLength, SQLSMALLINT *NameLength, - SQLSMALLINT *DataType, SQLULEN *ColumnSize, - SQLSMALLINT *DecimalDigits, SQLSMALLINT *Nullable) -{ - SQLRETURN r; - r = doSQLDescribeCol(StatementHandle, ColumnNumber, ColumnName, - BufferLength, NameLength, - DataType, ColumnSize, - DecimalDigits, Nullable); - return r; -} - -static SQLRETURN doSQLNumParams(SQLHSTMT hstmt, SQLSMALLINT *pcpar) -{ - sql_t *sql = (sql_t*)hstmt; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - if (!sql->stmt) { - SET_ERROR(sql, "HY010", TSDB_CODE_ODBC_STATEMENT_NOT_READY, ""); - return SQL_ERROR; - } - - int insert = 0; - int r = taos_stmt_is_insert(sql->stmt, &insert); - if (r) { - SET_ERROR(sql, "HY000", terrno, ""); - return SQL_ERROR; - } - // if (!insert) { - // SET_ERROR(sql, "HY000", terrno, "taos does not provide count of parameters for statement other than insert"); - // return SQL_ERROR; - // } - - int params = 0; - r = taos_stmt_num_params(sql->stmt, ¶ms); - if (r) { - SET_ERROR(sql, "HY000", terrno, "fetch num of statement params failed"); - return SQL_ERROR; - } - - if (pcpar) *pcpar = (SQLSMALLINT)params; - - return SQL_SUCCESS; -} - -SQLRETURN SQL_API SQLNumParams(SQLHSTMT hstmt, SQLSMALLINT *pcpar) -{ - SQLRETURN r; - r = doSQLNumParams(hstmt, pcpar); - return r; -} - -static SQLRETURN doSQLSetStmtAttr(SQLHSTMT StatementHandle, - SQLINTEGER Attribute, SQLPOINTER Value, - SQLINTEGER StringLength) -{ - sql_t *sql = (sql_t*)StatementHandle; - if (!sql) return SQL_INVALID_HANDLE; - - CHK_CONN(sql); - CHK_CONN_TAOS(sql); - - if (!sql->stmt) { - SET_ERROR(sql, "HY010", TSDB_CODE_ODBC_STATEMENT_NOT_READY, ""); - return SQL_ERROR; - } - - if (sql->is_executed) { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NOT_SUPPORT, "change attr after executing statement not supported yet"); - return SQL_ERROR; - } - - switch (Attribute) { - case SQL_ATTR_PARAM_BIND_TYPE: { - SQLULEN val = (SQLULEN)Value; - if (val==SQL_BIND_BY_COLUMN) { - sql->rowlen = 0; - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NOT_SUPPORT, "SQL_ATTR_PARAM_BIND_TYPE/SQL_BIND_BY_COLUMN"); - return SQL_ERROR; - } - sql->rowlen = val; - return SQL_SUCCESS; - } break; - case SQL_ATTR_PARAMSET_SIZE: { - SQLULEN val = (SQLULEN)Value; - DASSERT(val>0); - sql->n_rows = val; - return SQL_SUCCESS; - } break; - case SQL_ATTR_PARAM_BIND_OFFSET_PTR: { - if (Value) { - SQLULEN val = *(SQLULEN*)Value; - sql->ptr_offset = val; - } else { - sql->ptr_offset = 0; - } - return SQL_SUCCESS; - } break; - default: { - SET_ERROR(sql, "HY000", TSDB_CODE_ODBC_NOT_SUPPORT, "Attribute:%d", Attribute); - } break; - } - return SQL_ERROR; -} - -SQLRETURN SQL_API SQLSetStmtAttr(SQLHSTMT StatementHandle, - SQLINTEGER Attribute, SQLPOINTER Value, - SQLINTEGER StringLength) -{ - SQLRETURN r; - r = doSQLSetStmtAttr(StatementHandle, Attribute, Value, StringLength); - return r; -} - -#ifdef _MSC_VER - -#define POST_INSTALLER_ERROR(hwndParent, code, fmt, ...) \ -do { \ - char buf[4096]; \ - snprintf(buf, sizeof(buf), "%s[%d]%s():" fmt "", \ - basename((char*)__FILE__), __LINE__, __func__, \ - ##__VA_ARGS__); \ - SQLPostInstallerError(code, buf); \ - if (hwndParent) { \ - MessageBox(hwndParent, buf, "Error", MB_OK|MB_ICONEXCLAMATION); \ - } \ -} while (0) - -typedef struct kv_s kv_t; -struct kv_s { - char *line; - size_t val; -}; - -static BOOL get_driver_dll_path(HWND hwndParent, char *buf, size_t len) -{ - HMODULE hm = NULL; - - if (GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - (LPCSTR) &ConfigDSN, &hm) == 0) - { - int ret = GetLastError(); - POST_INSTALLER_ERROR(hwndParent, ODBC_ERROR_REQUEST_FAILED, "GetModuleHandle failed, error = %d\n", ret); - return FALSE; - } - if (GetModuleFileName(hm, buf, (DWORD)len) == 0) - { - int ret = GetLastError(); - POST_INSTALLER_ERROR(hwndParent, ODBC_ERROR_REQUEST_FAILED, "GetModuleFileName failed, error = %d\n", ret); - return FALSE; - } - return TRUE; -} - -static BOOL doDSNAdd(HWND hwndParent, LPCSTR lpszDriver, LPCSTR lpszAttributes) -{ - BOOL r = TRUE; - - kv_t *kvs = NULL; - - kv_t dsn = {0}; - char *line = NULL; - - do { - char driver_dll[MAX_PATH + 1]; - r = get_driver_dll_path(hwndParent, driver_dll, sizeof(driver_dll)); - if (!r) break; - - dsn.line = strdup("DSN=TAOS_DEMO"); - if (!dsn.line) { r = FALSE; break; } - - const char *p = lpszAttributes; - int ikvs = 0; - while (p && *p) { - line = strdup(p); - if (!line) { r = FALSE; break; } - char *v = strchr(line, '='); - if (!v) { r = FALSE; break; } - - if (strstr(line, "DSN")==line) { - if (dsn.line) { - free(dsn.line); - dsn.line = NULL; - dsn.val = 0; - } - dsn.line = line; - line = NULL; - } else { - kv_t *t = (kv_t*)realloc(kvs, (ikvs+1)*sizeof(*t)); - if (!t) { r = FALSE; free(line); break; } - t[ikvs].line = line; - *v = '\0'; - if (v) t[ikvs].val = v - line + 1; - line = NULL; - - kvs = t; - ++ikvs; - } - - p += strlen(p) + 1; - } - - if (hwndParent) { - MessageBox(hwndParent, "Please use odbcconf to add DSN for TAOS ODBC Driver", "Warning!", MB_OK|MB_ICONEXCLAMATION); - } - if (!r) break; - - char *v = NULL; - v = strchr(dsn.line, '='); - if (!v) { r = FALSE; break; } - *v = '\0'; - dsn.val = v - dsn.line + 1; - - if ((!dsn.line)) { - if (!r) POST_INSTALLER_ERROR(hwndParent, ODBC_ERROR_REQUEST_FAILED, "lack of either DSN or Driver"); - } else { - if (r) r = SQLWritePrivateProfileString("ODBC Data Sources", dsn.line+dsn.val, lpszDriver, "Odbc.ini"); - if (r) r = SQLWritePrivateProfileString(dsn.line+dsn.val, "Driver", driver_dll, "Odbc.ini"); - } - - for (int i=0; r && itype) { - case TSDB_DATA_TYPE_TINYINT: - return 5; - break; - - case TSDB_DATA_TYPE_SMALLINT: - return 7; - break; - - case TSDB_DATA_TYPE_INT: - return 12; - break; - - case TSDB_DATA_TYPE_BIGINT: - return 22; - break; - - case TSDB_DATA_TYPE_FLOAT: { - return 12; - } break; - - case TSDB_DATA_TYPE_DOUBLE: { - return 20; - } break; - - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: { - return 3*((size_t)field->bytes - VARSTR_HEADER_SIZE) + 2; - } break; - - case TSDB_DATA_TYPE_TIMESTAMP: - return 26; - break; - - case TSDB_DATA_TYPE_BOOL: - return 7; - default: - break; - } - - return 10; -} - - - diff --git a/src/connector/odbc/src/todbc.def b/src/connector/odbc/src/todbc.def index 1e080f01983ec2b38d657004291008f6da6198dc..2485e5df11490bafd5752e7fc3670ce09d47095a 100644 --- a/src/connector/odbc/src/todbc.def +++ b/src/connector/odbc/src/todbc.def @@ -3,13 +3,19 @@ SQLAllocEnv SQLFreeEnv SQLAllocConnect SQLFreeConnect +SQLGetEnvAttr +SQLSetEnvAttr +SQLGetConnectAttr +SQLGetConnectOption +SQLGetInfo SQLConnect SQLDisconnect SQLAllocStmt SQLAllocHandle +SQLFreeHandle SQLFreeStmt SQLExecDirect -SQLExecDirectW +;SQLExecDirectW SQLNumResultCols SQLRowCount SQLColAttribute @@ -17,14 +23,45 @@ SQLGetData SQLFetch SQLPrepare SQLExecute -SQLGetDiagField +SQLParamData +SQLPutData +;SQLGetDiagField SQLGetDiagRec SQLBindParameter +SQLDescribeParam SQLDriverConnect SQLSetConnectAttr SQLDescribeCol +SQLBindCol SQLNumParams SQLSetStmtAttr +SQLBindParam +SQLCancel +SQLCancelHandle +SQLCloseCursor +SQLColumns +SQLCopyDesc +SQLDataSources +SQLEndTran +;SQLError +SQLFetchScroll +SQLGetCursorName +SQLGetDescField +SQLGetDescRec +;SQLGetFunctions +SQLGetStmtAttr +SQLGetStmtOption +SQLGetTypeInfo +SQLSetConnectOption +SQLSetCursorName +SQLSetDescField +SQLSetDescRec +SQLSetParam +SQLSetStmtOption +SQLSpecialColumns +SQLStatistics +SQLTables +SQLTransact ConfigDSN ConfigTranslator ConfigDriver diff --git a/src/connector/odbc/src/todbc_buf.c b/src/connector/odbc/src/todbc_buf.c new file mode 100644 index 0000000000000000000000000000000000000000..1fce74299c53977359437dd1e43eba1f57eab936 --- /dev/null +++ b/src/connector/odbc/src/todbc_buf.c @@ -0,0 +1,410 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "todbc_buf.h" + +#include "todbc_list.h" +#include "todbc_log.h" +#include "todbc_tls.h" + +#include +#include + +#define BLOCK_SIZE (1024*16) + +// alignment for holding whatever struct would be +#define ALIGN(size) ((size==0?1:size) + sizeof(size_t) - 1) / sizeof(size_t) * sizeof(size_t); + +typedef struct buf_rec_s buf_rec_t; +typedef struct buf_block_s buf_block_t; + +struct buf_rec_s { + size_t size; // payload size couting from ptr[0] + buf_block_t *owner; + char ptr[0]; // place-holder +}; + +#define PTR_OFFSET() ((size_t)(((buf_rec_t*)0)->ptr)) + +struct buf_block_s { + todbc_buf_t *owner; + size_t cap; // total payload space available for allocation + size_t ptr; // beginning of free space + char base[0]; // place-holder +}; + +#define BASE_OFFSET() ((size_t)(((buf_block_t*)0)->base)) + +struct todbc_buf_s { + char buf[BLOCK_SIZE]; + buf_block_t *block; // mapped to &buf[0] + todbc_list_t *list; // dynamic allocated blocks list +}; + +#define BASE_STATIC(block) (block->owner->buf + BASE_OFFSET()==block->base) + + +typedef struct arg_s arg_t; +struct arg_s { + todbc_buf_t *buf; + + // input + char *arg_ptr; + size_t arg_size; + buf_rec_t *arg_rec; + size_t arg_align; + + // output + buf_rec_t *rec; +}; + +static buf_rec_t* ptr_rec(todbc_buf_t *buf, void *ptr) { + char *p = (char*)ptr; + char *begin = p-PTR_OFFSET(); + buf_rec_t *rec = (buf_rec_t*)begin; + OILE(rec, ""); + OILE(rec->size, ""); + OILE((rec->size)%sizeof(void*)==0, ""); + buf_block_t *block = rec->owner; + OILE(block, ""); + OILE(block->owner==buf, ""); + OILE(block->base, ""); + + char *end = begin + sizeof(buf_rec_t) + rec->size; + OILE(begin>=block->base, ""); + OILE(end<=block->base+block->ptr, ""); + + return rec; +} + +static buf_block_t* buf_block_create(size_t align); +static buf_rec_t* buf_block_realloc(buf_block_t *block, arg_t *arg); +static void buf_block_free(buf_block_t *block); +static void buf_block_reclaim(buf_block_t *block); + +#define ALLOC_LIST() do { \ + if (buf->list) break; \ + todbc_list_conf_t conf = {0}; \ + conf.arg = buf; \ + conf.val_free = do_free_buf_block; \ + buf->list = todbc_list_create(conf); \ +} while (0) + + +todbc_buf_t* todbc_buf_create(void) { + todbc_buf_t *buf = (todbc_buf_t*)calloc(1, sizeof(*buf)); + if (!buf) return NULL; + + buf_block_t *block = (buf_block_t*)(buf->buf); + + block->owner = buf; + block->cap = sizeof(buf->buf) - sizeof(buf_block_t); + block->ptr = 0; + + buf->block = block; + + OILE(BASE_STATIC(buf->block), ""); + + return buf; +} + +void todbc_buf_free(todbc_buf_t *buf) { + if (!buf) return; + + todbc_list_free(buf->list); + buf->list = NULL; + + free(buf); +} + +void* todbc_buf_alloc(todbc_buf_t *buf, size_t size) { + return todbc_buf_realloc(buf, NULL, size); +} + +void* todbc_buf_calloc(todbc_buf_t *buf, size_t count, size_t size) { + size_t align = ALIGN(size); + size_t total = count * align; + if (total > INT64_MAX) return NULL; + void *ptr = todbc_buf_realloc(buf, NULL, total); + if (!ptr) return NULL; + memset(ptr, 0, total); + return ptr; +} + +static void do_free_buf_block(todbc_list_t *list, void *val, void *arg); + +static int alloc_space_by(todbc_list_t *list, void *val, void *arg); + +void* todbc_buf_realloc(todbc_buf_t *buf, void *ptr, size_t size) { + OILE(buf, ""); + OILE(sizeof(buf_block_t)%sizeof(void*)==0, ""); + OILE(sizeof(buf_rec_t)%sizeof(void*)==0, ""); + + size_t align = ALIGN(size); + size_t req_size = align + sizeof(buf_rec_t); + + buf_rec_t *rec = NULL; + buf_block_t *block = NULL; + if (ptr) { + rec = ptr_rec(buf, ptr); + if (align<=rec->size) return ptr; + + block = rec->owner; + char *tail_rec = rec->ptr + rec->size; + char *tail_block = block->base + block->ptr; + if (tail_rec==tail_block) { + char *end_rec = rec->ptr + align; + char *end_block = block->base + block->cap; + if (end_rec<=end_block) { + rec->size = align; + block->ptr = (size_t)(end_rec - block->base); + return ptr; + } + } else { + size_t remain = block->cap - block->ptr; + if (req_size<=remain) { + char *new_ptr = block->base + block->ptr; + block->ptr += req_size; + buf_rec_t *new_rec = (buf_rec_t*)new_ptr; + new_rec->size = align; + new_rec->owner = block; + memcpy(new_rec->ptr, ptr, rec->size); + return new_rec->ptr; + } + } + } + + arg_t arg = {0}; + arg.buf = buf; + arg.arg_ptr = ptr; + arg.arg_size = size; + arg.arg_rec = rec; + arg.arg_align = align; + + if (block!=buf->block) { + buf_rec_t *new_rec = buf_block_realloc(buf->block, &arg); + if (new_rec) return new_rec->ptr; + } + + ALLOC_LIST(); + if (!buf->list) return NULL; + + int r = todbc_list_traverse(buf->list, alloc_space_by, &arg); + if (r) { + OILE(arg.rec, ""); + OILE(arg.rec->ptr, ""); + return arg.rec->ptr; + } + + block = buf_block_create(arg.arg_align); + if (!block) return NULL; + OILE(block->owner==NULL, ""); + r = todbc_list_pushfront(buf->list, block); + OILE(r==0, ""); + block->owner = buf; + buf_rec_t *p = buf_block_realloc(block, &arg); + OILE(p, ""); + + return p->ptr; +} + +static int do_reclaim(todbc_list_t *list, void *val, void *arg); + +void todbc_buf_reclaim(todbc_buf_t *buf) { + if (!buf) return; + + buf_block_reclaim(buf->block); + + if (!buf->list) return; + + todbc_list_traverse(buf->list, do_reclaim, buf); +} + +static int do_reclaim(todbc_list_t *list, void *val, void *arg) { + todbc_buf_t *buf = (todbc_buf_t*)arg; + buf_block_t *block = (buf_block_t*)val; + OILE(list, ""); + OILE(block, ""); + OILE(block->owner==buf, ""); + buf_block_reclaim(block); + return 0; +} + +static void buf_block_reclaim(buf_block_t *block) { + block->ptr = 0; +} + +static void buf_block_free(buf_block_t *block) { + if (!block) return; + + buf_block_reclaim(block); + + if (BASE_STATIC(block)) return; + + block->owner = NULL; + block->cap = 0; + block->ptr = 0; + + free(block); +} + +static void do_free_buf_block(todbc_list_t *list, void *val, void *arg) { + todbc_buf_t *buf = (todbc_buf_t*)arg; + OILE(buf && buf->list==list, ""); + buf_block_t *block = (buf_block_t*)val; + OILE(block, ""); + OILE(buf==block->owner, ""); + + buf_block_free(block); +} + +static int alloc_space_by(todbc_list_t *list, void *val, void *arg) { + buf_block_t *block = (buf_block_t*)val; + arg_t *targ = (arg_t*)arg; + + buf_rec_t *rec = targ->arg_rec; + if (rec && rec->owner == block) return 0; + + buf_rec_t *new_rec = buf_block_realloc(block, targ); + if (!new_rec) return 0; + return 1; +} + +static buf_block_t* buf_block_create(size_t size) { + size_t align = ALIGN(size); + size_t req_size = sizeof(buf_block_t) + sizeof(buf_rec_t) + align; + req_size = (req_size + BLOCK_SIZE - 1) / BLOCK_SIZE * BLOCK_SIZE; + + buf_block_t *block = (buf_block_t*)malloc(req_size); + if (!block) return NULL; + + block->owner = NULL; + block->cap = req_size - sizeof(buf_block_t); + block->ptr = 0; + + return block; +} + +static buf_rec_t* buf_block_realloc(buf_block_t *block, arg_t *arg) { + OILE(block, ""); + OILE(arg, ""); + OILE(block->base, ""); + OILE(block->cap >= block->ptr, ""); + + char *ptr = arg->arg_ptr; + buf_rec_t *rec = arg->arg_rec; + OILE(rec==NULL || rec->owner!=block, ""); + + size_t align = arg->arg_align; + size_t req_size = sizeof(buf_rec_t) + align; + + OILE(req_size>0, ""); + + size_t remain = block->cap - block->ptr; + if (req_size > remain) { + return NULL; + } + + char *p = block->base + block->ptr; + buf_rec_t *new_rec = (buf_rec_t*)p; + new_rec->size = align; + new_rec->owner = block; + + block->ptr += req_size; + + if (ptr) { + memcpy(new_rec->ptr, ptr, arg->arg_rec->size); + } + + arg->rec = new_rec; + + return new_rec; +} + + + + + + +// test + +void todbc_buf_test_random(size_t iterates, size_t size, int raw) { + size_t n_reallocs = 0; + todbc_buf_t *cache = NULL; + OD("use %s", raw ? "realloc" : "todbc_buf_t"); + if (!raw) { + cache = todbc_buf_create(); + OILE(cache, ""); + } + srand((unsigned)time(0)); + char *buf = NULL; + size_t blen = 0; + while (iterates-- > 0) { + char *p = NULL; + size_t i = 0; + size_t len = (size_t)rand()%size; + if (raw) { + p = realloc(buf, len); + } else { + p = todbc_buf_realloc(cache, buf, len); + } + OILE(p, ""); + for (i=blen; i 0) { + size_t i = 0; + char *buf = NULL; + while (i + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _todbc_buf_h_ +#define _todbc_buf_h_ + +#include +#include + +// non-thread-safe +typedef struct todbc_buf_s todbc_buf_t; + +todbc_buf_t* todbc_buf_create(void); +void todbc_buf_free(todbc_buf_t *buf); + +void* todbc_buf_alloc(todbc_buf_t *buf, size_t size); +void* todbc_buf_calloc(todbc_buf_t *buf, size_t count, size_t size); +void* todbc_buf_realloc(todbc_buf_t *buf, void *ptr, size_t size); +char* todbc_buf_strdup(todbc_buf_t *buf, const char *str); +void todbc_buf_reclaim(todbc_buf_t *buf); + +void todbc_buf_test_random(size_t iterates, size_t size, int raw); +void todbc_buf_test(size_t iterates, size_t size, int raw); + +#endif // _todbc_buf_h_ + diff --git a/src/connector/odbc/src/todbc_conv.c b/src/connector/odbc/src/todbc_conv.c deleted file mode 100644 index 9c0f19764c2b456c63b6d0a01402e37868a0a366..0000000000000000000000000000000000000000 --- a/src/connector/odbc/src/todbc_conv.c +++ /dev/null @@ -1,660 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "todbc_conv.h" - -#include "todbc_log.h" - -#include -#include -#include -#include -#include - -const char* tsdb_conv_code_str(TSDB_CONV_CODE code) { - switch (code) { - case TSDB_CONV_OK: return "TSDB_CONV_OK"; - case TSDB_CONV_NOT_AVAIL: return "TSDB_CONV_NOT_AVAIL"; - case TSDB_CONV_OOM: return "TSDB_CONV_OOM"; - case TSDB_CONV_OOR: return "TSDB_CONV_OOR"; - case TSDB_CONV_TRUNC_FRACTION: return "TSDB_CONV_TRUNC_FRACTION"; - case TSDB_CONV_TRUNC: return "TSDB_CONV_TRUNC"; - case TSDB_CONV_CHAR_NOT_NUM: return "TSDB_CONV_CHAR_NOT_NUM"; - case TSDB_CONV_CHAR_NOT_TS: return "TSDB_CONV_CHAR_NOT_TS"; - case TSDB_CONV_NOT_VALID_TS: return "TSDB_CONV_NOT_VALID_TS"; - case TSDB_CONV_GENERAL: return "TSDB_CONV_GENERAL"; - case TSDB_CONV_SRC_TOO_LARGE: return "TSDB_CONV_SRC_TOO_LARGE"; - case TSDB_CONV_SRC_BAD_SEQ: return "TSDB_CONV_SRC_BAD_SEQ"; - case TSDB_CONV_SRC_INCOMPLETE: return "TSDB_CONV_SRC_INCOMPLETE"; - case TSDB_CONV_SRC_GENERAL: return "TSDB_CONV_SRC_GENERAL"; - case TSDB_CONV_BAD_CHAR: return "TSDB_CONV_BAD_CHAR"; - default: return "UNKNOWN"; - }; -} - -// src: int -TSDB_CONV_CODE tsdb_int64_to_bit(int64_t src, int8_t *dst) { - *dst = (int8_t)src; - if (src==0 || src==1) return TSDB_CONV_OK; - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_int64_to_tinyint(int64_t src, int8_t *dst) { - *dst = (int8_t)src; - if (src == *dst) return TSDB_CONV_OK; - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_int64_to_smallint(int64_t src, int16_t *dst) { - *dst = (int16_t)src; - if (src == *dst) return TSDB_CONV_OK; - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_int64_to_int(int64_t src, int32_t *dst) { - *dst = (int32_t)src; - if (src == *dst) return TSDB_CONV_OK; - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_int64_to_bigint(int64_t src, int64_t *dst) { - *dst = src; - return TSDB_CONV_OK; -} - -TSDB_CONV_CODE tsdb_int64_to_ts(int64_t src, int64_t *dst) { - *dst = src; - - time_t t = (time_t)(src / 1000); - struct tm tm = {0}; - if (localtime_r(&t, &tm)) return TSDB_CONV_OK; - - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_int64_to_float(int64_t src, float *dst) { - *dst = (float)src; - - int64_t v = (int64_t)*dst; - if (v==src) return TSDB_CONV_OK; - - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_int64_to_double(int64_t src, double *dst) { - *dst = (double)src; - - int64_t v = (int64_t)*dst; - if (v==src) return TSDB_CONV_OK; - - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_int64_to_char(int64_t src, char *dst, size_t dlen) { - int n = snprintf(dst, dlen, "%" PRId64 "", src); - DASSERT(n>=0); - - if (n=2) return TSDB_CONV_OOR; - if (src == *dst) return TSDB_CONV_OK; - - int64_t v = (int64_t)src; - if (v == *dst) return TSDB_CONV_TRUNC_FRACTION; - - return TSDB_CONV_TRUNC; -} - -TSDB_CONV_CODE tsdb_double_to_tinyint(double src, int8_t *dst) { - *dst = (int8_t)src; - - if (srcSCHAR_MAX) return TSDB_CONV_OOR; - if (src == *dst) return TSDB_CONV_OK; - - int64_t v = (int64_t)src; - if (v == *dst) return TSDB_CONV_TRUNC_FRACTION; - - return TSDB_CONV_TRUNC; -} - -TSDB_CONV_CODE tsdb_double_to_smallint(double src, int16_t *dst) { - *dst = (int16_t)src; - - if (srcSHRT_MAX) return TSDB_CONV_OOR; - if (src == *dst) return TSDB_CONV_OK; - - int64_t v = (int64_t)src; - if (v == *dst) return TSDB_CONV_TRUNC_FRACTION; - - return TSDB_CONV_TRUNC; -} - -TSDB_CONV_CODE tsdb_double_to_int(double src, int32_t *dst) { - *dst = (int32_t)src; - - if (srcLONG_MAX) return TSDB_CONV_OOR; - if (src == *dst) return TSDB_CONV_OK; - - int64_t v = (int64_t)src; - if (v == *dst) return TSDB_CONV_TRUNC_FRACTION; - - return TSDB_CONV_TRUNC; -} - -TSDB_CONV_CODE tsdb_double_to_bigint(double src, int64_t *dst) { - *dst = (int64_t)src; - - if (srcLLONG_MAX) return TSDB_CONV_OOR; - if (src == *dst) return TSDB_CONV_OK; - - int64_t v = (int64_t)src; - if (v == *dst) return TSDB_CONV_TRUNC_FRACTION; - - return TSDB_CONV_TRUNC; -} - -TSDB_CONV_CODE tsdb_double_to_ts(double src, int64_t *dst) { - TSDB_CONV_CODE code = tsdb_double_to_bigint(src, dst); - - if (code==TSDB_CONV_OK || code==TSDB_CONV_TRUNC_FRACTION) { - int64_t v = (int64_t)src; - time_t t = (time_t)(v / 1000); - struct tm tm = {0}; - if (localtime_r(&t, &tm)) return TSDB_CONV_OK; - - return TSDB_CONV_OOR; - } - - return code; -} - -TSDB_CONV_CODE tsdb_double_to_char(double src, char *dst, size_t dlen) { - int n = snprintf(dst, dlen, "%lg", src); - DASSERT(n>=0); - - if (n=0); - if (n=19) return TSDB_CONV_TRUNC_FRACTION; - - return TSDB_CONV_TRUNC; -} - -// src: chars -TSDB_CONV_CODE tsdb_chars_to_bit(const char *src, size_t smax, int8_t *dst) { - if (strcmp(src, "0")==0) { - *dst = 0; - return TSDB_CONV_OK; - } - - if (strcmp(src, "1")==0) { - *dst = 1; - return TSDB_CONV_OK; - } - - double v; - int bytes; - int n = sscanf(src, "%lg%n", &v, &bytes); - - if (n!=1) return TSDB_CONV_CHAR_NOT_NUM; - if (bytes!=strlen(src)) return TSDB_CONV_CHAR_NOT_NUM; - - if (v<0 || v>=2) return TSDB_CONV_OOR; - - return TSDB_CONV_TRUNC_FRACTION; -} - -TSDB_CONV_CODE tsdb_chars_to_tinyint(const char *src, size_t smax, int8_t *dst) { - int64_t v; - TSDB_CONV_CODE code = tsdb_chars_to_bigint(src, smax, &v); - if (code!=TSDB_CONV_OK) return code; - - *dst = (int8_t)v; - - if (v==*dst) return TSDB_CONV_OK; - - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_chars_to_smallint(const char *src, size_t smax, int16_t *dst) { - int64_t v; - TSDB_CONV_CODE code = tsdb_chars_to_bigint(src, smax, &v); - if (code!=TSDB_CONV_OK) return code; - - *dst = (int16_t)v; - - if (v==*dst) return TSDB_CONV_OK; - - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_chars_to_int(const char *src, size_t smax, int32_t *dst) { - int64_t v; - TSDB_CONV_CODE code = tsdb_chars_to_bigint(src, smax, &v); - if (code!=TSDB_CONV_OK) return code; - - *dst = (int32_t)v; - - if (v==*dst) return TSDB_CONV_OK; - - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_chars_to_bigint(const char *src, size_t smax, int64_t *dst) { - int bytes; - int n = sscanf(src, "%" PRId64 "%n", dst, &bytes); - - if (n!=1) return TSDB_CONV_CHAR_NOT_NUM; - if (bytes==strlen(src)) { - return TSDB_CONV_OK; - } - - double v; - n = sscanf(src, "%lg%n", &v, &bytes); - if (n!=1) return TSDB_CONV_CHAR_NOT_NUM; - if (bytes==strlen(src)) { - return TSDB_CONV_TRUNC_FRACTION; - } - - return TSDB_CONV_OK; -} - -TSDB_CONV_CODE tsdb_chars_to_ts(const char *src, size_t smax, int64_t *dst) { - int64_t v; - TSDB_CONV_CODE code = tsdb_chars_to_bigint(src, smax, &v); - if (code!=TSDB_CONV_OK) return code; - - *dst = v; - - if (v==*dst) { - time_t t = (time_t)(v / 1000); - struct tm tm = {0}; - if (localtime_r(&t, &tm)) return TSDB_CONV_OK; - } - - return TSDB_CONV_OOR; -} - -TSDB_CONV_CODE tsdb_chars_to_float(const char *src, size_t smax, float *dst) { - int bytes; - int n = sscanf(src, "%g%n", dst, &bytes); - - if (n==1 && bytes==strlen(src)) { - return TSDB_CONV_OK; - } - - return TSDB_CONV_CHAR_NOT_NUM; -} - -TSDB_CONV_CODE tsdb_chars_to_double(const char *src, size_t smax, double *dst) { - int bytes; - int n = sscanf(src, "%lg%n", dst, &bytes); - - if (n==1 && bytes==strlen(src)) { - return TSDB_CONV_OK; - } - - return TSDB_CONV_CHAR_NOT_NUM; -} - -TSDB_CONV_CODE tsdb_chars_to_timestamp(const char *src, size_t smax, SQL_TIMESTAMP_STRUCT *dst) { - int64_t v = 0; - // why cast to 'char*' ? - int r = taosParseTime((char*)src, &v, (int32_t)smax, TSDB_TIME_PRECISION_MILLI, 0); - - if (r) { - return TSDB_CONV_CHAR_NOT_TS; - } - - time_t t = v/1000; - struct tm vtm = {0}; - localtime_r(&t, &vtm); - dst->year = (SQLSMALLINT)(vtm.tm_year + 1900); - dst->month = (SQLUSMALLINT)(vtm.tm_mon + 1); - dst->day = (SQLUSMALLINT)(vtm.tm_mday); - dst->hour = (SQLUSMALLINT)(vtm.tm_hour); - dst->minute = (SQLUSMALLINT)(vtm.tm_min); - dst->second = (SQLUSMALLINT)(vtm.tm_sec); - dst->fraction = (SQLUINTEGER)(v%1000 * 1000000); - - return TSDB_CONV_OK; -} - -TSDB_CONV_CODE tsdb_chars_to_timestamp_ts(const char *src, size_t smax, int64_t *dst) { - // why cast to 'char*' ? - int r = taosParseTime((char*)src, dst, (int32_t)smax, TSDB_TIME_PRECISION_MILLI, 0); - - if (r) { - return TSDB_CONV_CHAR_NOT_TS; - } - - return TSDB_CONV_OK; -} - -TSDB_CONV_CODE tsdb_chars_to_char(const char *src, size_t smax, char *dst, size_t dmax) { - int n = snprintf(dst, dmax, "%s", src); - DASSERT(n>=0); - if (nnext + bytes; - if (next>sizeof(buffer->buf)) return NULL; - - char *p = buffer->buf + buffer->next; - buffer->next = next; - return p; -} - -int is_owned_by_stack_buffer(stack_buffer_t *buffer, const char *ptr) { - if (!buffer) return 0; - if (ptr>=buffer->buf && ptrbuf+buffer->next) return 1; - return 0; -} - - -struct tsdb_conv_s { - iconv_t cnv; - unsigned int direct:1; -}; - -static tsdb_conv_t no_conversion = {0}; -static pthread_once_t once = PTHREAD_ONCE_INIT; -static void once_init(void) { - no_conversion.cnv = (iconv_t)-1; - no_conversion.direct = 1; -} - -tsdb_conv_t* tsdb_conv_direct() { // get a non-conversion-converter - pthread_once(&once, once_init); - return &no_conversion; -} - -tsdb_conv_t* tsdb_conv_open(const char *from_enc, const char *to_enc) { - pthread_once(&once, once_init); - tsdb_conv_t *cnv = (tsdb_conv_t*)calloc(1, sizeof(*cnv)); - if (!cnv) return NULL; - if (strcmp(from_enc, to_enc)==0 && 0) { - cnv->cnv = (iconv_t)-1; - cnv->direct = 1; - return cnv; - } - cnv->cnv = iconv_open(to_enc, from_enc); - if (cnv->cnv == (iconv_t)-1) { - free(cnv); - return NULL; - } - cnv->direct = 0; - return cnv; -} - -void tsdb_conv_close(tsdb_conv_t *cnv) { - if (!cnv) return; - if (cnv == &no_conversion) return; - if (!cnv->direct) { - if (cnv->cnv != (iconv_t)-1) { - iconv_close(cnv->cnv); - } - } - cnv->cnv = (iconv_t)-1; - cnv->direct = 0; - free(cnv); -} - -TSDB_CONV_CODE tsdb_conv_write(tsdb_conv_t *cnv, const char *src, size_t *slen, char *dst, size_t *dlen) { - if (!cnv) return TSDB_CONV_NOT_AVAIL; - if (cnv->direct) { - size_t n = (*slen > *dlen) ? *dlen : *slen; - memcpy(dst, src, n); - *slen -= n; - *dlen -= n; - if (*dlen) dst[n] = '\0'; - return TSDB_CONV_OK; - } - if (!cnv->cnv) return TSDB_CONV_NOT_AVAIL; - size_t r = iconv(cnv->cnv, (char**)&src, slen, &dst, dlen); - if (r==(size_t)-1) return TSDB_CONV_BAD_CHAR; - if (*slen) return TSDB_CONV_TRUNC; - if (*dlen) *dst = '\0'; - return TSDB_CONV_OK; -} - -TSDB_CONV_CODE tsdb_conv_write_int64(tsdb_conv_t *cnv, int64_t val, char *dst, size_t *dlen) { - char utf8[64]; - int n = snprintf(utf8, sizeof(utf8), "%" PRId64 "", val); - DASSERT(n>=0); - DASSERT(n=0); - DASSERT(n=0); - DASSERT(ndirect) { - if (src[slen]=='\0') { // access violation? - *dst = src; - if (dlen) *dlen = slen; - return TSDB_CONV_OK; - } - blen = slen + 1; - } else { - blen = (slen + 1) * 4; - } - - buf = stack_buffer_alloc(buffer, blen); - if (!buf) { - buf = (char*)malloc(blen); - if (!buf) return TSDB_CONV_OOM; - } - - if (cnv->direct) { - size_t n = slen; - DASSERT(blen > n); - memcpy(buf, src, n); - buf[n] = '\0'; - *dst = buf; - if (dlen) *dlen = n; - return TSDB_CONV_OK; - } - - const char *orig_s = src; - char *orig_d = buf; - size_t orig_blen = blen; - - TSDB_CONV_CODE code; - size_t r = iconv(cnv->cnv, (char**)&src, &slen, &buf, &blen); - do { - if (r==(size_t)-1) { - switch(errno) { - case E2BIG: { - code = TSDB_CONV_SRC_TOO_LARGE; - } break; - case EILSEQ: { - code = TSDB_CONV_SRC_BAD_SEQ; - } break; - case EINVAL: { - code = TSDB_CONV_SRC_INCOMPLETE; - } break; - default: { - code = TSDB_CONV_SRC_GENERAL; - } break; - } - break; - } - if (slen) { - code = TSDB_CONV_TRUNC; - break; - } - DASSERT(blen); - *buf = '\0'; - *dst = orig_d; - if (dlen) *dlen = orig_blen - blen; - return TSDB_CONV_OK; - } while (0); - - if (orig_d!=(char*)orig_s && !is_owned_by_stack_buffer(buffer, orig_d)) free(orig_d); - return code; -} - -void tsdb_conv_free(tsdb_conv_t *cnv, const char *ptr, stack_buffer_t *buffer, const char *src) { - if (ptr!=src && !is_owned_by_stack_buffer(buffer, ptr)) free((char*)ptr); -} - diff --git a/src/connector/odbc/src/todbc_conv.h b/src/connector/odbc/src/todbc_conv.h deleted file mode 100644 index 2941f3e4961d38ed1e72bfd3d1184d1ea8de251b..0000000000000000000000000000000000000000 --- a/src/connector/odbc/src/todbc_conv.h +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _todbc_conv_h_ -#define _todbc_conv_h_ - -#include "os.h" -#include -#include - - -typedef enum { - TSDB_CONV_OK = 0, - TSDB_CONV_NOT_AVAIL, - TSDB_CONV_OOM, - TSDB_CONV_OOR, - TSDB_CONV_TRUNC_FRACTION, - TSDB_CONV_TRUNC, - TSDB_CONV_CHAR_NOT_NUM, - TSDB_CONV_CHAR_NOT_TS, - TSDB_CONV_NOT_VALID_TS, - TSDB_CONV_GENERAL, - TSDB_CONV_BAD_CHAR, - TSDB_CONV_SRC_TOO_LARGE, - TSDB_CONV_SRC_BAD_SEQ, - TSDB_CONV_SRC_INCOMPLETE, - TSDB_CONV_SRC_GENERAL, -} TSDB_CONV_CODE; - -const char* tsdb_conv_code_str(TSDB_CONV_CODE code); - -typedef struct stack_buffer_s stack_buffer_t; -struct stack_buffer_s { - char buf[1024*16]; - size_t next; -}; - -char* stack_buffer_alloc(stack_buffer_t *buffer, size_t bytes); -int is_owned_by_stack_buffer(stack_buffer_t *buffer, const char *ptr); - -typedef struct tsdb_conv_s tsdb_conv_t; -tsdb_conv_t* tsdb_conv_direct(); // get a non-conversion-converter -tsdb_conv_t* tsdb_conv_open(const char *from_enc, const char *to_enc); -void tsdb_conv_close(tsdb_conv_t *cnv); - -TSDB_CONV_CODE tsdb_conv_write(tsdb_conv_t *cnv, const char *src, size_t *slen, char *dst, size_t *dlen); -TSDB_CONV_CODE tsdb_conv_write_int64(tsdb_conv_t *cnv, int64_t val, char *dst, size_t *dlen); -TSDB_CONV_CODE tsdb_conv_write_double(tsdb_conv_t *cnv, double val, char *dst, size_t *dlen); -TSDB_CONV_CODE tsdb_conv_write_timestamp(tsdb_conv_t *cnv, SQL_TIMESTAMP_STRUCT val, char *dst, size_t *dlen); - -TSDB_CONV_CODE tsdb_conv_chars_to_bit(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, int8_t *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_tinyint(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, int8_t *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_smallint(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, int16_t *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_int(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, int32_t *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_bigint(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, int64_t *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_ts(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, int64_t *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_float(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, float *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_double(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, double *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_timestamp(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, SQL_TIMESTAMP_STRUCT *dst); -TSDB_CONV_CODE tsdb_conv_chars_to_timestamp_ts(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, int64_t *dst); -TSDB_CONV_CODE tsdb_conv(tsdb_conv_t *cnv, stack_buffer_t *buffer, const char *src, size_t slen, const char **dst, size_t *dlen); -void tsdb_conv_free(tsdb_conv_t *cnv, const char *ptr, stack_buffer_t *buffer, const char *src); - - -TSDB_CONV_CODE tsdb_int64_to_bit(int64_t src, int8_t *dst); -TSDB_CONV_CODE tsdb_int64_to_tinyint(int64_t src, int8_t *dst); -TSDB_CONV_CODE tsdb_int64_to_smallint(int64_t src, int16_t *dst); -TSDB_CONV_CODE tsdb_int64_to_int(int64_t src, int32_t *dst); -TSDB_CONV_CODE tsdb_int64_to_bigint(int64_t src, int64_t *dst); -TSDB_CONV_CODE tsdb_int64_to_ts(int64_t src, int64_t *dst); -TSDB_CONV_CODE tsdb_int64_to_float(int64_t src, float *dst); -TSDB_CONV_CODE tsdb_int64_to_double(int64_t src, double *dst); -TSDB_CONV_CODE tsdb_int64_to_char(int64_t src, char *dst, size_t dlen); - -TSDB_CONV_CODE tsdb_double_to_bit(double src, int8_t *dst); -TSDB_CONV_CODE tsdb_double_to_tinyint(double src, int8_t *dst); -TSDB_CONV_CODE tsdb_double_to_smallint(double src, int16_t *dst); -TSDB_CONV_CODE tsdb_double_to_int(double src, int32_t *dst); -TSDB_CONV_CODE tsdb_double_to_bigint(double src, int64_t *dst); -TSDB_CONV_CODE tsdb_double_to_ts(double src, int64_t *dst); -TSDB_CONV_CODE tsdb_double_to_char(double src, char *dst, size_t dlen); - -TSDB_CONV_CODE tsdb_timestamp_to_char(SQL_TIMESTAMP_STRUCT src, char *dst, size_t dlen); - -TSDB_CONV_CODE tsdb_chars_to_bit(const char *src, size_t smax, int8_t *dst); -TSDB_CONV_CODE tsdb_chars_to_tinyint(const char *src, size_t smax, int8_t *dst); -TSDB_CONV_CODE tsdb_chars_to_smallint(const char *src, size_t smax, int16_t *dst); -TSDB_CONV_CODE tsdb_chars_to_int(const char *src, size_t smax, int32_t *dst); -TSDB_CONV_CODE tsdb_chars_to_bigint(const char *src, size_t smax, int64_t *dst); -TSDB_CONV_CODE tsdb_chars_to_ts(const char *src, size_t smax, int64_t *dst); -TSDB_CONV_CODE tsdb_chars_to_float(const char *src, size_t smax, float *dst); -TSDB_CONV_CODE tsdb_chars_to_double(const char *src, size_t smax, double *dst); -TSDB_CONV_CODE tsdb_chars_to_timestamp(const char *src, size_t smax, SQL_TIMESTAMP_STRUCT *dst); -TSDB_CONV_CODE tsdb_chars_to_char(const char *src, size_t smax, char *dst, size_t dmax); - -#endif // _todbc_conv_h_ - diff --git a/src/connector/odbc/src/todbc_flex.h b/src/connector/odbc/src/todbc_flex.h index a13f1f4d2ebd8bbca73d9ff224bb3ed20ed43174..762ffba0bec7bcfd47f45ad4475d08f20dc2375a 100644 --- a/src/connector/odbc/src/todbc_flex.h +++ b/src/connector/odbc/src/todbc_flex.h @@ -16,16 +16,40 @@ #ifndef _TODBC_FLEX_H_ #define _TODBC_FLEX_H_ +#include +#include + +// TSDB predefined field types +// TINYINT SMALLINT INT BIGINT FLOAT DOUBLE BOOL TIMESTAMP BINARY NCHAR + +typedef struct map_tsdb_type_s map_tsdb_type_t; +struct map_tsdb_type_s { + SQLSMALLINT tsdb_tinyint; + SQLSMALLINT tsdb_smallint; + SQLSMALLINT tsdb_int; + SQLSMALLINT tsdb_bigint; + SQLULEN tsdb_bigint_size; + SQLSMALLINT tsdb_float; + SQLSMALLINT tsdb_double; + SQLSMALLINT tsdb_bool; + SQLSMALLINT tsdb_timestamp; + SQLSMALLINT tsdb_binary; + SQLSMALLINT tsdb_nchar; +}; + typedef struct conn_val_s conn_val_t; struct conn_val_s { - char *key; - char *dsn; - char *uid; - char *pwd; - char *db; - char *server; - char *svr_enc; - char *cli_enc; + char *dsn; + char *uid; + char *pwd; + char *db; + char *server; + char *enc_local; + char *enc_char; + char *enc_wchar; + char *enc_db; + + map_tsdb_type_t tsdb_map; }; diff --git a/src/connector/odbc/src/todbc_hash.c b/src/connector/odbc/src/todbc_hash.c new file mode 100644 index 0000000000000000000000000000000000000000..1f64a490fd2adf5688e320c14d22db8c279c9f98 --- /dev/null +++ b/src/connector/odbc/src/todbc_hash.c @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "todbc_hash.h" + +#include "todbc_list.h" +#include "todbc_log.h" + +#include + +typedef struct todbc_hash_slot_s todbc_hash_slot_t; + +struct todbc_hash_s { + todbc_hash_conf_t conf; + + todbc_hash_slot_t *slots; + size_t n_slots; +}; + +struct todbc_hash_slot_s { + todbc_list_t *list; +}; + +todbc_hash_t* todbc_hash_create(todbc_hash_conf_t conf) { + if (!conf.key_hash) return NULL; + if (!conf.key_comp) return NULL; + if (!conf.val_free) return NULL; + + todbc_hash_t *hash = (todbc_hash_t*)calloc(1, sizeof(*hash)); + if (!hash) return NULL; + + hash->conf = conf; + if (hash->conf.slots==0) { + hash->conf.slots = 33; + } + + hash->slots = (todbc_hash_slot_t*)calloc(hash->conf.slots, sizeof(*hash->slots)); + do { + if (!hash->slots) break; + hash->n_slots = hash->conf.slots; + return hash; + } while (0); + + todbc_hash_free(hash); + return NULL; +} + +void todbc_hash_free(todbc_hash_t *hash) { + if (!hash) return; + for (int i=0; in_slots; ++i) { + todbc_hash_slot_t *slot = hash->slots + i; + if (!slot->list) continue; + todbc_list_free(slot->list); + slot->list = NULL; + } + free(hash->slots); + hash->n_slots = 0; +} + +typedef struct kv_s kv_t; +struct kv_s { + todbc_hash_t *hash; + void *key; + void *val; +}; + +static void do_val_free(todbc_list_t *list, void *val, void *arg) { + todbc_hash_t *hash = (todbc_hash_t*)arg; + DASSERT(list); + DASSERT(hash); + DASSERT(hash->conf.val_free); + hash->conf.val_free(hash, val, hash->conf.arg); +} + +int todbc_hash_put(todbc_hash_t *hash, void *key, void *val) { + if (!hash) return -1; + if (!hash->slots) return -1; + if (!key) return -1; + if (!val) return -1; + + void *old = NULL; + int found = 0; + int r = todbc_hash_get(hash, key, &old, &found); + if (r) return r; + if (found) return -1; + + unsigned long hash_val = hash->conf.key_hash(hash, key); + todbc_hash_slot_t *slot = hash->slots + (hash_val % hash->n_slots); + if (!slot->list) { + todbc_list_conf_t conf = {0}; + conf.arg = hash; + conf.val_free = do_val_free; + + slot->list = todbc_list_create(conf); + if (!slot->list) return -1; + } + + r = todbc_list_pushback(slot->list, val); + + return r; +} + +static int do_comp(todbc_list_t *list, void *old, void *val, void *arg) { + kv_t *kv = (kv_t*)arg; + DASSERT(kv); + DASSERT(kv->hash); + DASSERT(kv->key); + DASSERT(kv->hash->conf.key_comp); + DASSERT(kv->key == val); + int r = kv->hash->conf.key_comp(kv->hash, kv->key, old); + if (r==0) { + kv->val = old; + } + return r; +} + +int todbc_hash_get(todbc_hash_t *hash, void *key, void **val, int *found) { + if (!hash) return -1; + if (!hash->slots) return -1; + if (!key) return -1; + if (!val) return -1; + if (!found) return -1; + + *found = 0; + + unsigned long hash_val = hash->conf.key_hash(hash, key); + todbc_hash_slot_t *slot = hash->slots + (hash_val % hash->n_slots); + if (slot->list) { + kv_t kv = {0}; + kv.hash = hash; + kv.key = key; + kv.val = NULL; + int r = todbc_list_find(slot->list, key, found, do_comp, &kv); + if (*found) { + DASSERT(r==0); + DASSERT(kv.val); + *val = kv.val; + } + return r; + } + + return 0; +} + +int todbc_hash_del(todbc_hash_t *hash, void *key) { + return -1; +} + +typedef struct arg_s arg_t; +struct arg_s { + todbc_hash_t *hash; + void *arg; + int (*iterate)(todbc_hash_t *hash, void *val, void *arg); +}; + +static int do_iterate(todbc_list_t *list, void *val, void *arg); + +int todbc_hash_traverse(todbc_hash_t *hash, int (*iterate)(todbc_hash_t *hash, void *val, void *arg), void *arg) { + if (!hash) return -1; + if (!iterate) return -1; + + for (int i=0; in_slots; ++i) { + todbc_hash_slot_t *slot = hash->slots + i; + if (!slot->list) continue; + arg_t targ = {0}; + targ.hash = hash; + targ.arg = arg; + targ.iterate = iterate; + int r = todbc_list_traverse(slot->list, do_iterate, &targ); + if (r) return r; + } + + return 0; +} + +static int do_iterate(todbc_list_t *list, void *val, void *arg) { + arg_t *targ = (arg_t*)arg; + DASSERT(targ); + DASSERT(targ->iterate); + DASSERT(targ->hash); + return targ->iterate(targ->hash, val, targ->arg); +} + diff --git a/src/connector/odbc/src/todbc_hash.h b/src/connector/odbc/src/todbc_hash.h new file mode 100644 index 0000000000000000000000000000000000000000..9172c8b7f9c79b84f643d25bb317593675e15521 --- /dev/null +++ b/src/connector/odbc/src/todbc_hash.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _todbc_hash_h_ +#define _todbc_hash_h_ + + +#include +#include + +// non-thread-safe + +typedef struct todbc_hash_conf_s todbc_hash_conf_t; +typedef struct todbc_hash_s todbc_hash_t; + +typedef void (*hash_val_free_f)(todbc_hash_t *hash, void *val, void *arg); +typedef unsigned long (*hash_key_hash_f)(todbc_hash_t *hash, void *key); +typedef int (*hash_key_comp_f)(todbc_hash_t *hash, void *key, void *val); + +struct todbc_hash_conf_s { + void *arg; + hash_val_free_f val_free; + + hash_key_hash_f key_hash; + hash_key_comp_f key_comp; + + size_t slots; +}; + +todbc_hash_t* todbc_hash_create(todbc_hash_conf_t conf); +void todbc_hash_free(todbc_hash_t *hash); + +// fail if key exists +int todbc_hash_put(todbc_hash_t *hash, void *key, void *val); +int todbc_hash_get(todbc_hash_t *hash, void *key, void **val, int *found); +int todbc_hash_del(todbc_hash_t *hash, void *key); +typedef int (*hash_iterate_f)(todbc_hash_t *hash, void *val, void *arg); +int todbc_hash_traverse(todbc_hash_t *hash, hash_iterate_f iterate, void *arg); + +#endif // _todbc_hash_h_ + diff --git a/src/connector/odbc/src/todbc_iconv.c b/src/connector/odbc/src/todbc_iconv.c new file mode 100644 index 0000000000000000000000000000000000000000..968cc870f81d420484de3361a54e8185901c3f14 --- /dev/null +++ b/src/connector/odbc/src/todbc_iconv.c @@ -0,0 +1,682 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "todbc_iconv.h" + +#include "todbc_hash.h" +#include "todbc_log.h" +#include "todbc_string.h" +#include "todbc_tls.h" + +#define invalid_iconv() ((iconv_t)-1) + +struct todbc_iconvset_s { + todbc_hash_t *iconv_hash; + + todbc_hash_t *enc_hash; +}; + +typedef struct todbc_iconv_key_s todbc_iconv_key_t; +struct todbc_iconv_key_s { + const char *enc_to; + const char *enc_from; +}; + +struct todbc_iconv_s { + char enc_to[64]; + char enc_from[64]; + todbc_iconv_key_t key; + iconv_t cnv; + + todbc_enc_t from; + todbc_enc_t to; + + unsigned int direct:1; +}; + +static void todbc_iconv_free(todbc_iconv_t *val); + +todbc_iconvset_t* todbc_iconvset_create(void) { + todbc_iconvset_t *cnvset = (todbc_iconvset_t*)calloc(1, sizeof(*cnvset)); + if (!cnvset) return NULL; + + return cnvset; +} + +void todbc_iconvset_free(todbc_iconvset_t *cnvset) { + if (!cnvset) return; + if (cnvset->iconv_hash) { + todbc_hash_free(cnvset->iconv_hash); + cnvset->iconv_hash = NULL; + } + if (cnvset->enc_hash) { + todbc_hash_free(cnvset->enc_hash); + cnvset->enc_hash = NULL; + } + free(cnvset); +} + +static void do_iconv_hash_val_free(todbc_hash_t *hash, void *val, void *arg); +static unsigned long do_iconv_hash_key_hash(todbc_hash_t *hash, void *key); +static int do_iconv_hash_key_comp(todbc_hash_t *hash, void *key, void *val); + +static void do_enc_hash_val_free(todbc_hash_t *hash, void *val, void *arg); +static unsigned long do_enc_hash_key_hash(todbc_hash_t *hash, void *key); +static int do_enc_hash_key_comp(todbc_hash_t *hash, void *key, void *val); + +#define CHK(x, y, n) if (strcasecmp(x, y)==0) return n + +#define SET_SIZES(cs, enc, a,b) do { \ + if (strcasecmp(enc->enc, cs)==0) { \ + enc->char_size = a; \ + enc->variable_char_size = b; \ + return; \ + } \ +} while (0) + +static void do_set_sizes(todbc_enc_t *enc) { + if (!enc) return; + + SET_SIZES("ISO-10646-UCS-2", enc, 2, -1); + SET_SIZES("UCS-2", enc, 2, -1); + SET_SIZES("CSUNICODE", enc, 2, -1); + SET_SIZES("UCS-2BE", enc, 2, -1); + SET_SIZES("UNICODE-1-1", enc, 2, -1); + SET_SIZES("UNICODEBIG", enc, 2, -1); + SET_SIZES("CSUNICODE11", enc, 2, -1); + SET_SIZES("UCS-2LE", enc, 2, -1); + SET_SIZES("UNICODELITTLE", enc, 2, -1); + SET_SIZES("UTF-16", enc, 2, -1); + SET_SIZES("UTF-16BE", enc, 2, -1); + SET_SIZES("UTF-16LE", enc, 2, -1); + SET_SIZES("UCS-2-INTERNAL", enc, 2, -1); + SET_SIZES("UCS-2-SWAPPED", enc, 2, -1); + SET_SIZES("ISO-10646-UCS-4", enc, 4, -1); + SET_SIZES("UCS-4", enc, 4, -1); + SET_SIZES("CSUCS4", enc, 4, -1); + SET_SIZES("UCS-4BE", enc, 4, -1); + SET_SIZES("UCS-4LE", enc, 4, -1); + SET_SIZES("UTF-32", enc, 4, -1); + SET_SIZES("UTF-32BE", enc, 4, -1); + SET_SIZES("UTF-32LE", enc, 4, -1); + SET_SIZES("UCS-4-INTERNAL", enc, 4, -1); + SET_SIZES("UCS-4-SWAPPED", enc, 4, -1); + + SET_SIZES("UTF-8", enc, -1, 3); + SET_SIZES("UTF8", enc, -1, 3); + SET_SIZES("UTF-8-MAC", enc, -1, 3); + SET_SIZES("UTF8-MAC", enc, -1, 3); + + SET_SIZES("CN-GB", enc, 2, -1); + SET_SIZES("EUC-CN", enc, 2, -1); + SET_SIZES("EUCCN", enc, 2, -1); + SET_SIZES("GB2312", enc, 2, -1); + SET_SIZES("CSGB2312", enc, 2, -1); + SET_SIZES("GBK", enc, 2, -1); + SET_SIZES("CP936", enc, 2, -1); + SET_SIZES("MS936", enc, 2, -1); + SET_SIZES("WINDOWS-936", enc, 2, -1); + SET_SIZES("GB18030", enc, 2, -1); + + // add more setup here after + + enc->char_size = -1; + enc->variable_char_size = -1; +} + +static int do_get_null_size(const char *enc, int *null_size); + +// static int do_get_unicode_char_size(const char *enc) { +// if (!enc) return -1; +// +// CHK("ISO-10646-UCS-2", enc, 2); +// CHK("UCS-2", enc, 2); +// CHK("CSUNICODE", enc, 2); +// CHK("UCS-2BE", enc, 2); +// CHK("UNICODE-1-1", enc, 2); +// CHK("UNICODEBIG", enc, 2); +// CHK("CSUNICODE11", enc, 2); +// CHK("UCS-2LE", enc, 2); +// CHK("UNICODELITTLE", enc, 2); +// CHK("UTF-16", enc, 2); +// CHK("UTF-16BE", enc, 2); +// CHK("UTF-16LE", enc, 2); +// CHK("UCS-2-INTERNAL", enc, 2); +// CHK("UCS-2-SWAPPED", enc, 2); +// CHK("ISO-10646-UCS-4", enc, 4); +// CHK("UCS-4", enc, 4); +// CHK("CSUCS4", enc, 4); +// CHK("UCS-4BE", enc, 4); +// CHK("UCS-4LE", enc, 4); +// CHK("UTF-32", enc, 4); +// CHK("UTF-32BE", enc, 4); +// CHK("UTF-32LE", enc, 4); +// CHK("UCS-4-INTERNAL", enc, 4); +// CHK("UCS-4-SWAPPED", enc, 4); +// +// return -1; +// } + +todbc_enc_t todbc_iconvset_enc(todbc_iconvset_t *cnvset, const char *enc) { + do { + if (!cnvset) break; + if (!enc) break; + + if (!cnvset->enc_hash) { + todbc_hash_conf_t conf = {0}; + conf.arg = cnvset; + conf.val_free = do_enc_hash_val_free; + conf.key_hash = do_enc_hash_key_hash; + conf.key_comp = do_enc_hash_key_comp; + conf.slots = 7; + cnvset->enc_hash = todbc_hash_create(conf); + if (!cnvset->enc_hash) break; + } + + void *old = NULL; + int found = 0; + int r = todbc_hash_get(cnvset->enc_hash, (void*)enc, &old, &found); + if (r) { + DASSERT(found==0); + DASSERT(old==NULL); + break; + } + + if (found) { + DASSERT(old); + todbc_enc_t *val = (todbc_enc_t*)old; + return *val; + } + + todbc_enc_t *val = (todbc_enc_t*)calloc(1, sizeof(*val)); + if (!val) break; + do { + if (snprintf(val->enc, sizeof(val->enc), "%s", enc)>=sizeof(val->enc)) { + break; + } + do_set_sizes(val); + + if (do_get_null_size(val->enc, &val->null_size)) break; + + return *val; + } while (0); + + free(val); + } while (0); + + todbc_enc_t v = {0}; + v.char_size = -1; + v.null_size = -1; + + return v; +} + +todbc_iconv_t* todbc_iconvset_get(todbc_iconvset_t *cnvset, const char *enc_to, const char *enc_from) { + if (!cnvset) return NULL; + if (!enc_to) return NULL; + if (!enc_from) return NULL; + todbc_iconv_key_t key; + key.enc_to = enc_to; + key.enc_from = enc_from; + + if (!cnvset->iconv_hash) { + todbc_hash_conf_t conf = {0}; + conf.arg = cnvset; + conf.val_free = do_iconv_hash_val_free; + conf.key_hash = do_iconv_hash_key_hash; + conf.key_comp = do_iconv_hash_key_comp; + conf.slots = 7; + cnvset->iconv_hash = todbc_hash_create(conf); + if (!cnvset->iconv_hash) return NULL; + } + + void *old = NULL; + int found = 0; + int r = todbc_hash_get(cnvset->iconv_hash, &key, &old, &found); + if (r) { + DASSERT(found==0); + DASSERT(old==NULL); + return NULL; + } + + if (found) { + DASSERT(old); + todbc_iconv_t *val = (todbc_iconv_t*)old; + // D("found [%p] for [%s->%s]", val, enc_from, enc_to); + return val; + } + + todbc_iconv_t *val = (todbc_iconv_t*)calloc(1, sizeof(*val)); + if (!val) return NULL; + do { + if (snprintf(val->enc_to, sizeof(val->enc_to), "%s", enc_to)>=sizeof(val->enc_to)) { + break; + } + if (snprintf(val->enc_from, sizeof(val->enc_from), "%s", enc_from)>=sizeof(val->enc_from)) { + break; + } + val->key.enc_to = val->enc_to; + val->key.enc_from = val->enc_from; + if (strcasecmp(enc_to, enc_from)==0) { + val->direct = 1; + } + + val->from = todbc_tls_iconv_enc(enc_from); + val->to = todbc_tls_iconv_enc(enc_to); + + val->cnv = iconv_open(enc_to, enc_from); + if (val->cnv==invalid_iconv()) break; + + r = todbc_hash_put(cnvset->iconv_hash, &key, val); + + if (r) break; + + // D("created [%p] for [%s->%s]", val, enc_from, enc_to); + return val; + } while (0); + + todbc_iconv_free(val); + return NULL; +} + +iconv_t todbc_iconv_get(todbc_iconv_t *cnv) { + if (!cnv) return invalid_iconv(); + return cnv->cnv; +} + +// static int todbc_legal_chars_by_cnv(iconv_t cnv, const unsigned char *str, todbc_bytes_t *bc); + +int todbc_iconv_get_legal_chars(todbc_iconv_t *cnv, const unsigned char *str, todbc_bytes_t *bc) { + DASSERT(0); + // if (!cnv) return -1; + // if (bc->inbytes==0 || bc->inbytes > INT64_MAX) { + // DASSERT(bc->chars<=INT64_MAX); + // if (bc->chars > 0) { + // DASSERT(cnv->from_char_size==2 || cnv->from_char_size==4); + // bc->inbytes = ((size_t)cnv->from_char_size) * bc->chars; + // bc->chars = 0; + // } + // } else { + // DASSERT(bc->chars==0); + // } + // return todbc_legal_chars_by_cnv(todbc_iconv_get(cnv), str, bc); +} + +// static int todbc_legal_chars_by_block(iconv_t cnv, const unsigned char *str, todbc_bytes_t *bc); + +// static int todbc_legal_chars_by_cnv(iconv_t cnv, const unsigned char *str, todbc_bytes_t *bc) { +// size_t max_bytes = bc->inbytes; +// if (max_bytes > INT64_MAX) max_bytes = (size_t)-1; +// +// if (max_bytes != (size_t)-1) { +// return todbc_legal_chars_by_block(cnv, str, bc); +// } +// +// memset(bc, 0, sizeof(*bc)); +// +// size_t nbytes = 0; +// size_t ch_bytes = 0; +// +// char buf[16]; +// char *inbuf = (char*)str; +// char *outbuf; +// size_t outbytes; +// size_t inbytes; +// +// size_t n = 0; +// +// int r = 0; +// inbytes = 1; +// while (1) { +// if (nbytes==max_bytes) break; +// outbytes = sizeof(buf); +// outbuf = buf; +// +// ch_bytes = inbytes; +// n = iconv(cnv, &inbuf, &inbytes, &outbuf, &outbytes); +// nbytes += 1; +// if (n==(size_t)-1) { +// int err = errno; +// if (err!=EINVAL) { +// E("......."); +// r = -1; +// break; +// } +// inbytes = ch_bytes + 1; +// continue; +// } +// DASSERT(inbytes==0); +// n = sizeof(buf) - outbytes; +// +// DASSERT(n==2); +// if (buf[0]=='\0' && buf[1]=='\0') { +// ch_bytes = 0; +// break; +// } +// +// bc->inbytes += ch_bytes; +// bc->chars += 1; +// bc->outbytes += 2; +// +// ch_bytes = 0; +// inbytes = 1; +// } +// +// outbytes = sizeof(buf); +// outbuf = buf; +// n = iconv(cnv, NULL, NULL, &outbuf, &outbytes); +// +// if (r) return -1; +// if (n==(size_t)-1) return -1; +// if (outbytes!=sizeof(buf)) return -1; +// if (outbuf!=buf) return -1; +// if (ch_bytes) return -1; +// return 0; +// } + +unsigned char* todbc_iconv_conv(todbc_iconv_t *cnv, todbc_buf_t *buf, const unsigned char *src, todbc_bytes_t *bc) { + if (!buf) { + buf = todbc_tls_buf(); + if (!buf) return NULL; + } + if (bc==NULL) { + todbc_bytes_t x = {0}; + x.inbytes = (size_t)-1; + return todbc_iconv_conv(cnv, buf, src, &x); + } + + DASSERT(buf); + DASSERT(cnv); + DASSERT(src); + + todbc_string_t s = todbc_string_init(cnv->from.enc, src, bc->inbytes); + // D("total_bytes/bytes: %d/%zu/%zu", s.total_bytes, s.bytes); + DASSERT(s.buf==src); + todbc_string_t t = todbc_string_conv_to(&s, cnv->to.enc, NULL); + DASSERT(t.buf); + // bc->outbytes not counting size of null-terminator + bc->outbytes = t.bytes; + return (unsigned char*)t.buf; +} + +size_t todbc_iconv_est_bytes(todbc_iconv_t *cnv, size_t inbytes) { + DASSERT(cnv); + DASSERT(inbytes<=INT64_MAX); + const todbc_enc_t *from = &cnv->from; + const todbc_enc_t *to = &cnv->to; + + size_t outbytes = inbytes; + do { + if (from == to) break; + + size_t inchars = inbytes; + if (from->char_size > 1) { + inchars = (inbytes + (size_t)from->char_size - 1) / (size_t)from->char_size; + } + outbytes = inchars; + size_t char_size = MAX_CHARACTER_SIZE; + if (to->char_size > 0) { + char_size = (size_t)to->char_size; + } else if (to->variable_char_size > 0) { + char_size = (size_t)to->variable_char_size; + } + outbytes *= char_size; + } while (0); + + size_t nullbytes = MAX_CHARACTER_SIZE; + if (to->null_size > 0) { + nullbytes = (size_t)to->null_size; + } + // D("%s->%s: %zu->%zu", from->enc, to->enc, inbytes, outbytes); + outbytes += nullbytes; + // D("%s->%s: %zu->%zu", from->enc, to->enc, inbytes, outbytes); + + return outbytes; +} + +size_t todbc_iconv_bytes(todbc_iconv_t *cnv, size_t inchars) { + DASSERT(cnv); + if (inchars >= INT64_MAX) return (size_t)-1; + const todbc_enc_t *from = &cnv->from; + if (from->char_size > 0) { + return inchars * (size_t)from->char_size; + } + return (size_t)-1; +} + +todbc_enc_t todbc_iconv_from(todbc_iconv_t *cnv) { + DASSERT(cnv); + return cnv->from; +} + +todbc_enc_t todbc_iconv_to(todbc_iconv_t *cnv) { + DASSERT(cnv); + return cnv->to; +} + +int todbc_iconv_raw(todbc_iconv_t *cnv, const unsigned char *src, size_t *slen, unsigned char *dst, size_t *dlen) { + DASSERT(cnv); + DASSERT(src && dst); + DASSERT(slen && *slen < INT64_MAX); + DASSERT(dlen && *dlen < INT64_MAX); + + const int null_bytes = todbc_iconv_to(cnv).null_size; + + if (*dlen<=null_bytes) { + D("target buffer too small to hold even null-terminator"); + *dlen = 0; // while slen does not change + return -1; + } + + char *inbuf = (char*)src; + size_t inbytes = *slen; + char *outbuf = (char*)dst; + size_t outbytes = *dlen; + + size_t n = iconv(cnv->cnv, &inbuf, &inbytes, &outbuf, &outbytes); + int e = 0; + if (n==(size_t)-1) { + e = errno; + D("iconv failed: [%s->%s]:[%d]%s", cnv->from.enc, cnv->to.enc, e, strerror(e)); + } else { + DASSERT(n==0); + } + + const size_t inremain = inbytes; + const size_t outremain = outbytes; + *slen = inremain; + *dlen = outremain; + + // writing null-terminator to make dest a real string + DASSERT(outbytes <= INT64_MAX); + if (outbytes < null_bytes) { + D("target buffer too small to hold null-terminator"); + return -1; + } else { + for (int i=0; icnv, NULL, NULL, NULL, NULL); + + return 0; +} + +todbc_string_t todbc_iconv_conv2(todbc_iconv_t *cnv, todbc_buf_t *buf, const unsigned char *src, size_t *slen) { + const todbc_string_t nul = {0}; + if (!buf) { + buf = todbc_tls_buf(); + if (!buf) return nul; + } + DASSERT(cnv); + DASSERT(src); + + size_t inbytes = (size_t)-1; + if (slen && *slen <= INT64_MAX) inbytes = *slen; + + todbc_string_t in = todbc_string_init(todbc_iconv_from(cnv).enc, src, inbytes); + if (in.buf!=src) return nul; + if (in.bytes > INT64_MAX) return nul; + inbytes = in.bytes; + + size_t outblock = todbc_iconv_est_bytes(cnv, in.bytes); + if (outblock > INT64_MAX) return nul; + + unsigned char *out = todbc_buf_alloc(buf, outblock); + if (!out) return nul; + + const unsigned char *inbuf = src; + unsigned char *outbuf = out; + + + size_t outbytes = outblock; + int r = todbc_iconv_raw(cnv, inbuf, &inbytes, outbuf, &outbytes); + if (slen) *slen = inbytes; + if (r) return nul; + todbc_string_t s = {0}; + s.buf = outbuf; + s.bytes = outblock - outbytes; + s.total_bytes = s.bytes; + return s; +} + +// static int todbc_legal_chars_by_block(iconv_t cnv, const unsigned char *str, todbc_bytes_t *bc) { +// int r = 0; +// size_t max_bytes = bc->inbytes; +// char buf[1024*16]; +// memset(bc, 0, sizeof(*bc)); +// char *inbuf = (char*)str; +// while (bc->inbytesinbytes; +// size_t outbytes = sizeof(buf); +// char *outbuf = buf; +// size_t n = iconv(cnv, &inbuf, &inbytes, &outbuf, &outbytes); +// int err = 0; +// if (n==(size_t)-1) { +// err = errno; +// if (err!=E2BIG && err!=EINVAL) r = -1; +// } else { +// DASSERT(n==0); +// } +// if (inbytes == max_bytes - bc->inbytes) { +// r = -1; +// } +// bc->inbytes += max_bytes - bc->inbytes - inbytes; +// bc->outbytes += sizeof(buf) - outbytes; +// if (r) break; +// } +// bc->chars = bc->outbytes / 2; +// iconv(cnv, NULL, NULL, NULL, NULL); +// return r ? -1 : 0; +// } + +static void todbc_iconv_free(todbc_iconv_t *val) { + if (!val) return; + if (val->cnv!=invalid_iconv()) { + iconv_close(val->cnv); + val->cnv = invalid_iconv(); + } + free(val); +} + +// http://www.cse.yorku.ca/~oz/hash.html +static const unsigned long hash_seed = 5381; +static unsigned long hashing(unsigned long hash, const unsigned char *str); + +static void do_enc_hash_val_free(todbc_hash_t *hash, void *val, void *arg) { + todbc_iconvset_t *cnv = (todbc_iconvset_t*)arg; + todbc_enc_t *v = (todbc_enc_t*)val; + DASSERT(hash); + DASSERT(cnv); + DASSERT(v); + free(v); +} + +static unsigned long do_enc_hash_key_hash(todbc_hash_t *hash, void *key) { + const char *k = (const char*)key; + DASSERT(k); + unsigned long h = hash_seed; + h = hashing(h, (const unsigned char*)k); + return h; +} + +static int do_enc_hash_key_comp(todbc_hash_t *hash, void *key, void *val) { + const char *k = (const char*)key; + todbc_enc_t *v = (todbc_enc_t*)val; + if (strcasecmp(k, v->enc)) return 1; + return 0; +} + +static int do_get_null_size(const char *enc, int *null_size) { + iconv_t cnv = iconv_open(enc, UTF8_ENC); + if (cnv==(iconv_t)-1) return -1; + + char src[] = ""; + char dst[64]; + + char *inbuf = src; + size_t inbytes = 1; + char *outbuf = dst; + size_t outbytes = sizeof(dst); + + size_t n = iconv(cnv, &inbuf, &inbytes, &outbuf, &outbytes); + DASSERT(n==0); + DASSERT(inbytes==0); + + int size = (int)(sizeof(dst) - outbytes); + + iconv_close(cnv); + + if (null_size) *null_size = size; + + return 0; +} + +static void do_iconv_hash_val_free(todbc_hash_t *hash, void *val, void *arg) { + todbc_iconvset_t *cnv = (todbc_iconvset_t*)arg; + todbc_iconv_t *v = (todbc_iconv_t*)val; + DASSERT(hash); + DASSERT(cnv); + DASSERT(v); + todbc_iconv_free(v); +} + +static unsigned long do_iconv_hash_key_hash(todbc_hash_t *hash, void *key) { + todbc_iconv_key_t *k = (todbc_iconv_key_t*)key; + DASSERT(k); + unsigned long h = hash_seed; + h = hashing(h, (const unsigned char*)k->enc_to); + h = hashing(h, (const unsigned char*)k->enc_from); + return h; +} + +static int do_iconv_hash_key_comp(todbc_hash_t *hash, void *key, void *val) { + todbc_iconv_key_t *k = (todbc_iconv_key_t*)key; + todbc_iconv_t *v = (todbc_iconv_t*)val; + if (strcasecmp(k->enc_to, v->key.enc_to)) return 1; + if (strcasecmp(k->enc_from, v->key.enc_from)) return 1; + return 0; +} + +static unsigned long hashing(unsigned long hash, const unsigned char *str) { + unsigned long c; + while ((c = *str++)!=0) { + hash = ((hash << 5) + hash) + c; /* hash * 33 + c */ + } + + return hash; +} + diff --git a/src/connector/odbc/src/todbc_iconv.h b/src/connector/odbc/src/todbc_iconv.h new file mode 100644 index 0000000000000000000000000000000000000000..269900f8f1fc77da2bdf6dd4e298ed765e5e7e9e --- /dev/null +++ b/src/connector/odbc/src/todbc_iconv.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _todbc_iconv_h_ +#define _todbc_iconv_h_ + +#include "todbc_buf.h" +#include "todbc_string.h" + +#include + +// non-thread-safe + +#define ASCII_ENC "ASCII" +#define UTF8_ENC "UTF-8" +#define UTF16_ENC "UCS-2LE" +#define UNICODE_ENC "UCS-4LE" +#define GB18030_ENC "GB18030" + +#define MAX_CHARACTER_SIZE 6 + +typedef enum { + TSDB_CONV_OK = 0, + TSDB_CONV_NOT_AVAIL, + TSDB_CONV_OOM, + TSDB_CONV_OOR, + TSDB_CONV_TRUNC_FRACTION, + TSDB_CONV_TRUNC, + TSDB_CONV_CHAR_NOT_NUM, + TSDB_CONV_CHAR_NOT_TS, + TSDB_CONV_NOT_VALID_TS, + TSDB_CONV_GENERAL, + TSDB_CONV_BAD_CHAR, + TSDB_CONV_SRC_TOO_LARGE, + TSDB_CONV_SRC_BAD_SEQ, + TSDB_CONV_SRC_INCOMPLETE, + TSDB_CONV_SRC_GENERAL, +} TSDB_CONV_CODE; + +typedef struct todbc_iconvset_s todbc_iconvset_t; +typedef struct todbc_iconv_s todbc_iconv_t; +typedef struct todbc_enc_s todbc_enc_t; +struct todbc_enc_s { + int char_size; // character size at most + int null_size; // size for null terminator + int variable_char_size; // such as 3 for UTF8 + char enc[64]; // move here to satisfy todbc_enc_t enc = {0}; +}; + +todbc_iconvset_t* todbc_iconvset_create(void); +void todbc_iconvset_free(todbc_iconvset_t *cnvset); +todbc_iconv_t* todbc_iconvset_get(todbc_iconvset_t *cnvset, const char *enc_to, const char *enc_from); +todbc_enc_t todbc_iconvset_enc(todbc_iconvset_t *cnvset, const char *enc); + +typedef struct todbc_bytes_s todbc_bytes_t; +typedef struct todbc_err_s todbc_err_t; + +struct todbc_err_s { + unsigned int einval:1; // EINVAL + unsigned int eilseq:1; // EILSEQ + unsigned int etoobig:1; // E2BIG + unsigned int eoom:1; // ENOMEM +}; + +struct todbc_bytes_s { + size_t inbytes, outbytes; + size_t chars; + + todbc_err_t err; +}; + +typedef struct todbc_iconv_arg_s todbc_iconv_arg_t; +struct todbc_iconv_arg_s { + const unsigned char *inbuf; + size_t inbytes; // -1: not set + unsigned char *outbuf; + size_t outbytes; // -1: not set + + size_t chars; // -1: not set +}; + +iconv_t todbc_iconv_get(todbc_iconv_t *cnv); +int todbc_iconv_get_legal_chars(todbc_iconv_t *cnv, const unsigned char *str, todbc_bytes_t *bc); +// non-thread-safe +// use todbc_buf_t as mem-allocator, if NULL, fall-back to thread-local version +unsigned char* todbc_iconv_conv(todbc_iconv_t *cnv, todbc_buf_t *buf, const unsigned char *src, todbc_bytes_t *bc); +// null-terminator-inclusive +size_t todbc_iconv_est_bytes(todbc_iconv_t *cnv, size_t inbytes); +// if inchars>=0 && enc_from has fixed-char-size, returns inchars * char_size +// otherwise -1 +size_t todbc_iconv_bytes(todbc_iconv_t *cnv, size_t inchars); +todbc_enc_t todbc_iconv_from(todbc_iconv_t *cnv); +todbc_enc_t todbc_iconv_to(todbc_iconv_t *cnv); + +// at return, *slen/*dlen stores the remaining # +int todbc_iconv_raw(todbc_iconv_t *cnv, const unsigned char *src, size_t *slen, unsigned char *dst, size_t *dlen); +// use todbc_buf_t as mem-allocator, if NULL, fall-back to thread-local version +// at return, *slen stores the remaining # +todbc_string_t todbc_iconv_conv2(todbc_iconv_t *cnv, todbc_buf_t *buf, const unsigned char *src, size_t *slen); + +#endif // _todbc_iconv_h_ + diff --git a/src/connector/odbc/src/todbc_list.c b/src/connector/odbc/src/todbc_list.c new file mode 100644 index 0000000000000000000000000000000000000000..447ce4826c4d6417af3c7d983a9278c8c6adf32c --- /dev/null +++ b/src/connector/odbc/src/todbc_list.c @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "todbc_list.h" + +#include "todbc_log.h" + +#include + +struct todbc_list_s { + todbc_list_conf_t conf; + + todbc_list_node_t *head; + todbc_list_node_t *tail; + size_t count; +}; + +struct todbc_list_node_s { + void *val; + + todbc_list_t *list; + todbc_list_node_t *next; + todbc_list_node_t *prev; +}; + +static void do_remove(todbc_list_t *list, todbc_list_node_t *node); + +static void do_pushback(todbc_list_t *list, todbc_list_node_t *node); +static void do_pushfront(todbc_list_t *list, todbc_list_node_t *node); + +todbc_list_t* todbc_list_create(todbc_list_conf_t conf) { + todbc_list_t *list = (todbc_list_t*)calloc(1, sizeof(*list)); + if (!list) return NULL; + + list->conf = conf; + + return list; +} + +void todbc_list_free(todbc_list_t *list) { + if (!list) return; + + while (list->head) { + void *val = list->head->val; + do_remove(list, list->head); + if (!list->conf.val_free) continue; + list->conf.val_free(list, val, list->conf.arg); + } + + DASSERT(list->count == 0); +} + +size_t todbc_list_count(todbc_list_t *list) { + if (!list) return 0; + return list->count; +} + +int todbc_list_pushback(todbc_list_t *list, void *val) { + if (!list) return -1; + + todbc_list_node_t *node = (todbc_list_node_t*)calloc(1, sizeof(*node)); + if (!node) return -1; + node->val = val; + + do_pushback(list, node); + + return 0; +} + +int todbc_list_pushfront(todbc_list_t *list, void *val) { + if (!list) return -1; + + todbc_list_node_t *node = (todbc_list_node_t*)calloc(1, sizeof(*node)); + if (!node) return -1; + node->val = val; + + do_pushfront(list, node); + + return 0; +} + +int todbc_list_popback(todbc_list_t *list, void **val, int *found) { + if (!list) return -1; + if (!found) return -1; + + *found = 0; + + todbc_list_node_t *node = list->tail; + if (!node) return 0; + + if (val) *val = node->val; + do_remove(list, node); + + *found = 1; + + return 0; +} + +int todbc_list_popfront(todbc_list_t *list, void **val, int *found) { + if (!list) return -1; + if (!found) return -1; + + *found = 0; + + todbc_list_node_t *node = list->head; + if (!node) return 0; + + if (val) *val = node->val; + do_remove(list, node); + + *found = 1; + + return 0; +} + +int todbc_list_pop(todbc_list_t *list, void *val, int *found) { + if (!list) return -1; + if (!found) return -1; + + *found = 0; + + todbc_list_node_t *node = list->head; + while (node) { + if (node->val == val) break; + node = node->next; + } + + if (!node) return 0; + + do_remove(list, node); + + *found = 1; + + return 0; +} + +int todbc_list_traverse(todbc_list_t *list, list_iterate_f iterate, void *arg) { + if (!list) return -1; + if (!iterate) return -1; + + todbc_list_node_t *node = list->head; + while (node) { + int r = iterate(list, node->val, arg); + if (r) return r; + node = node->next; + } + + return 0; +} + +typedef struct comp_s comp_t; +struct comp_s { + list_comp_f comp; + void *arg; + int found; + void *val; +}; + +static int do_comp(todbc_list_t *list, void *val, void *arg); + +int todbc_list_find(todbc_list_t *list, void *val, int *found, list_comp_f comp, void *arg) { + if (!list) return -1; + if (!found) return -1; + if (!comp) return -1; + + *found = 0; + + comp_t sarg = {0}; + sarg.comp = comp; + sarg.arg = arg; + sarg.val = val; + todbc_list_traverse(list, do_comp, &sarg); + if (sarg.found) { + *found = 1; + } + return 0; +} + +static int do_comp(todbc_list_t *list, void *val, void *arg) { + comp_t *sarg = (comp_t*)arg; + + int r = sarg->comp(list, val, sarg->val, sarg->arg); + if (r==0) { + sarg->found = 1; + } + + return r; +} + +static void do_remove(todbc_list_t *list, todbc_list_node_t *node) { + DASSERT(node); + DASSERT(list); + DASSERT(list == node->list); + + todbc_list_node_t *prev = node->prev; + todbc_list_node_t *next = node->next; + if (prev) prev->next = next; + else list->head = next; + if (next) next->prev = prev; + else list->tail = prev; + node->prev = NULL; + node->next = NULL; + node->list = NULL; + node->val = NULL; + + list->count -= 1; + DASSERT(list->count <= INT64_MAX); + + free(node); +} + +static void do_pushback(todbc_list_t *list, todbc_list_node_t *node) { + DASSERT(list); + DASSERT(node); + DASSERT(node->list == NULL); + DASSERT(node->prev == NULL); + DASSERT(node->next == NULL); + + node->list = list; + node->prev = list->tail; + if (list->tail) list->tail->next = node; + else list->head = node; + list->tail = node; + + list->count += 1; + DASSERT(list->count > 0); +} + +static void do_pushfront(todbc_list_t *list, todbc_list_node_t *node) { + DASSERT(node); + DASSERT(node->list == NULL); + DASSERT(node->prev == NULL); + DASSERT(node->next == NULL); + + node->list = list; + node->next = list->head; + if (list->head) list->head->prev = node; + else list->tail = node; + list->head = node; + + list->count += 1; + DASSERT(list->count > 0); +} + diff --git a/src/connector/odbc/src/todbc_list.h b/src/connector/odbc/src/todbc_list.h new file mode 100644 index 0000000000000000000000000000000000000000..d84ffd88a0e8d06cea2b7382da8cba5b881813a1 --- /dev/null +++ b/src/connector/odbc/src/todbc_list.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _todbc_list_h_ +#define _todbc_list_h_ + +#include +#include + +// non-thread-safe + +typedef struct todbc_list_s todbc_list_t; +typedef struct todbc_list_node_s todbc_list_node_t; + +typedef struct todbc_list_conf_s todbc_list_conf_t; + +typedef void (*list_val_free_f)(todbc_list_t *list, void *val, void *arg); + +struct todbc_list_conf_s { + void *arg; + list_val_free_f val_free; +}; + +todbc_list_t* todbc_list_create(todbc_list_conf_t conf); +void todbc_list_free(todbc_list_t *list); + +size_t todbc_list_count(todbc_list_t *list); + +int todbc_list_pushback(todbc_list_t *list, void *val); +int todbc_list_pushfront(todbc_list_t *list, void *val); +int todbc_list_popback(todbc_list_t *list, void **val, int *found); +int todbc_list_popfront(todbc_list_t *list, void **val, int *found); +int todbc_list_pop(todbc_list_t *list, void *val, int *found); +typedef int (*list_iterate_f)(todbc_list_t *list, void *val, void *arg); +int todbc_list_traverse(todbc_list_t *list, list_iterate_f iterate, void *arg); +typedef int (*list_comp_f)(todbc_list_t *list, void *old, void *val, void *arg); +int todbc_list_find(todbc_list_t *list, void *val, int *found, list_comp_f comp, void *arg); + + +#endif // _todbc_list_h_ + diff --git a/src/connector/odbc/src/todbc_log.c b/src/connector/odbc/src/todbc_log.c new file mode 100644 index 0000000000000000000000000000000000000000..c83ad2d62df3ce08cb70bd035898ec9e33b893fe --- /dev/null +++ b/src/connector/odbc/src/todbc_log.c @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "todbc_log.h" + +uint64_t todbc_get_threadid(void) { + uint64_t tid = 0; +#ifdef __APPLE__ + pthread_threadid_np(NULL, &tid); +#elif defined(__linux__) + tid = (uint64_t)syscall(__NR_gettid); +#elif defined(_MSC_VER) + tid = GetCurrentThreadId(); +#else +#error you have to check the target API for thread id +#endif + return tid; +} + +#ifdef _MSC_VER +static __declspec(thread) uint64_t thread_id = 0; +#else +static __thread uint64_t thread_id = 0; +#endif + +#ifdef __GNUC__ + __attribute__((format(printf, 6, 7))) +#endif +void todbc_log(const char *file, int line, const char *func, const char tag, int err, const char *fmt, ...) { + struct tm stm = {0}; + struct timeval tv; + + if (thread_id==0) { + thread_id = todbc_get_threadid(); + } + + gettimeofday(&tv, NULL); + time_t tt = tv.tv_sec; + localtime_r(&tt, &stm); + + char buf[4096]; + size_t bytes = sizeof(buf); + + char *p = buf; + int n = 0; + + n = snprintf(p, bytes, "%C%02d:%02d:%02d.%06d[%" PRIx64 "]%s[%d]%s()", + tag, stm.tm_hour, stm.tm_min, stm.tm_sec, (int)tv.tv_usec, + thread_id, basename((char*)file), line, func); + if (n>0) { + bytes -= (size_t)n; + p += n; + } + + if (bytes>0) { + if (tag=='E' && err) { + n = snprintf(p, bytes, "[%d]%s", err, strerror(err)); + if (n>0) { + bytes -= (size_t)n; + p += n; + } + } + } + + if (bytes>0) { + n = snprintf(p, bytes, ": "); + if (n>0) { + bytes -= (size_t)n; + p += n; + } + } + + if (bytes>0) { + va_list ap; + va_start(ap, fmt); + n = vsnprintf(p, bytes, fmt, ap); + va_end(ap); + } + + fprintf(stderr, "%s\n", buf); +} + diff --git a/src/connector/odbc/src/todbc_log.h b/src/connector/odbc/src/todbc_log.h index 391a690cccb0954736cac76af3354cc8a39754a8..30ef8436e1e02535f9a7a3eaefc456a792bd1744 100644 --- a/src/connector/odbc/src/todbc_log.h +++ b/src/connector/odbc/src/todbc_log.h @@ -18,25 +18,35 @@ #include "os.h" -#define D(fmt, ...) \ - fprintf(stderr, \ - "%s[%d]:%s() " fmt "\n", \ - basename((char*)__FILE__), __LINE__, __func__, \ - ##__VA_ARGS__) - -#define DASSERT(statement) \ -do { \ - if (statement) break; \ - D("Assertion failure: %s", #statement); \ - abort(); \ -} while (0) +#ifdef __GNUC__ + __attribute__((format(printf, 6, 7))) +#endif +void todbc_log(const char *file, int line, const char *func, const char tag, int err, const char *fmt, ...); -#define DASSERTX(statement, fmt, ...) \ -do { \ - if (statement) break; \ - D("Assertion failure: %s, " fmt "", #statement, ##__VA_ARGS__); \ - abort(); \ +#define OL(tag, err, fmt, ...) todbc_log(__FILE__, __LINE__, __func__, tag, err, "%s" fmt "", "", ##__VA_ARGS__) +#define OD(fmt, ...) OL('D', 0, fmt, ##__VA_ARGS__) +#define OE(fmt, ...) OL('E', errno, fmt, ##__VA_ARGS__) +#define OW(fmt, ...) OL('W', 0, fmt, ##__VA_ARGS__) +#define OI(fmt, ...) OL('I', 0, fmt, ##__VA_ARGS__) +#define OV(fmt, ...) OL('V', 0, fmt, ##__VA_ARGS__) +#define OA(statement, fmt, ...) do { \ + if (statement) break; \ + OL('A', 0, "Assertion failure:[%s]; " fmt "", #statement, ##__VA_ARGS__); \ + abort(); \ } while (0) +#define OILE(statement, fmt, ...) OA(statement, "internal logic error: [" fmt "]", ##__VA_ARGS__) +#define ONIY(statement, fmt, ...) OA(statement, "not implemented yet: [" fmt "]", ##__VA_ARGS__) +#define ONSP(statement, fmt, ...) OA(statement, "not support yet: [" fmt "]", ##__VA_ARGS__) + +#define D(fmt, ...) OD(fmt, ##__VA_ARGS__) +#define E(fmt, ...) OE(fmt, ##__VA_ARGS__) +#define DASSERT(statement) OA(statement, "") +#define DASSERTX(statement, fmt, ...) OA(statement, fmt, ##__VA_ARGS__) + + +uint64_t todbc_get_threadid(void); + + #endif // _todbc_log_h_ diff --git a/src/connector/odbc/src/todbc_scanner.l b/src/connector/odbc/src/todbc_scanner.l index f8c6a15d92442ee7f7b9041f017b41a4ed590314..b36c894f73cd7d8eb3b1642eb8b9c3bb433f2a5b 100644 --- a/src/connector/odbc/src/todbc_scanner.l +++ b/src/connector/odbc/src/todbc_scanner.l @@ -1,11 +1,18 @@ %{ +#ifdef _MSC_VER +#include +#define strncasecmp _strnicmp +#define strcasecmp _stricmp +#define basename PathFindFileNameA +#else +#include +#endif + #include "todbc_flex.h" #include +#include -#ifdef _MSC_VER -#define strncasecmp _strnicmp -#define strcasecmp _stricmp -#endif +static int process_map(const char *cfg, map_tsdb_type_t *tsdb_map, int type); #define PUSH_STATE(state) yy_push_state(state, yyscanner) #define POP_STATE() yy_pop_state(yyscanner) @@ -28,50 +35,73 @@ do { \ while (yyleng) unput(yytext[yyleng-1]); \ } while (0) -#define set_key() \ -do { \ - free(yyextra->key); \ - yyextra->key = strdup(yytext); \ +#define set_val() \ +do { \ + int r = 0; \ + int curr; TOP_STATE(curr); \ + POP_STATE(); \ + int state; TOP_STATE(state); \ + switch(state) { \ + case DSN: { \ + free(yyextra->dsn); \ + yyextra->dsn = strdup(yytext); \ + } break; \ + case UID: { \ + free(yyextra->uid); \ + yyextra->uid = strdup(yytext); \ + } break; \ + case PWD: { \ + free(yyextra->pwd); \ + yyextra->pwd = strdup(yytext); \ + } break; \ + case SERVER: { \ + free(yyextra->server); \ + yyextra->server = strdup(yytext); \ + } break; \ + case DB: { \ + free(yyextra->db); \ + yyextra->db = strdup(yytext); \ + } break; \ + case ENC_CHAR: { \ + free(yyextra->enc_char); \ + yyextra->enc_char = strdup(yytext); \ + } break; \ + case ENC_WCHAR: { \ + free(yyextra->enc_wchar); \ + yyextra->enc_wchar = strdup(yytext); \ + } break; \ + case ENC_DB: { \ + free(yyextra->enc_db); \ + yyextra->enc_db = strdup(yytext); \ + } break; \ + case ENC_LOCAL: { \ + free(yyextra->enc_local); \ + yyextra->enc_local = strdup(yytext); \ + } break; \ + case TSDB_FLOAT: { \ + if (process_map(yytext, &yyextra->tsdb_map, TSDB_FLOAT)) { \ + r = -1; \ + } \ + } break; \ + case TSDB_BIGINT: { \ + if (process_map(yytext, &yyextra->tsdb_map, TSDB_BIGINT)) { \ + r = -1; \ + } \ + } break; \ + case KEY: { \ + } break; \ + default: { \ + r = -1; \ + } break; \ + } \ + PUSH_STATE(curr); \ + if (r) return r; \ } while (0) -#define set_val() \ -do { \ - if (!yyextra->key) break; \ - if (strcasecmp(yyextra->key, "DSN")==0) { \ - free(yyextra->dsn); \ - yyextra->dsn = strdup(yytext); \ - break; \ - } \ - if (strcasecmp(yyextra->key, "UID")==0) { \ - free(yyextra->uid); \ - yyextra->uid = strdup(yytext); \ - break; \ - } \ - if (strcasecmp(yyextra->key, "PWD")==0) { \ - free(yyextra->pwd); \ - yyextra->pwd = strdup(yytext); \ - break; \ - } \ - if (strcasecmp(yyextra->key, "DB")==0) { \ - free(yyextra->db); \ - yyextra->pwd = strdup(yytext); \ - break; \ - } \ - if (strcasecmp(yyextra->key, "Server")==0) { \ - free(yyextra->server); \ - yyextra->server = strdup(yytext); \ - break; \ - } \ - if (strcasecmp(yyextra->key, "SERVER_ENC")==0) { \ - free(yyextra->svr_enc); \ - yyextra->svr_enc = strdup(yytext); \ - break; \ - } \ - if (strcasecmp(yyextra->key, "CLIENT_ENC")==0) { \ - free(yyextra->cli_enc); \ - yyextra->cli_enc = strdup(yytext); \ - break; \ - } \ +#define FAIL() \ +do { \ + /*fprintf(stderr, "==%s[%d]%s()==\n", basename(__FILE__), __LINE__, __func__);*/ \ + return -1; \ } while (0) %} @@ -89,57 +119,85 @@ do { \ %option warn %option perf-report %option 8bit +%option case-insensitive +%x DSN UID PWD SERVER DB +%x ENC_CHAR ENC_WCHAR ENC_DB ENC_LOCAL +%x TSDB_FLOAT TSDB_BIGINT %x KEY EQ BRACE1 BRACE2 VAL %% <> { int state; TOP_STATE(state); if (state == INITIAL) yyterminate(); if (state == VAL) yyterminate(); - return -1; } + FAIL(); } [[:space:]]+ { } -[[:alnum:]_]+ { set_key(); PUSH_STATE(KEY); } -.|\n { return -1; } +"DSN" { PUSH_STATE(DSN); } +"UID" { PUSH_STATE(UID); } +"PWD" { PUSH_STATE(PWD); } +"Server" { PUSH_STATE(SERVER); } +"DB" { PUSH_STATE(DB); } +"ENC_CHAR" { PUSH_STATE(ENC_CHAR); } +"ENC_WCHAR" { PUSH_STATE(ENC_WCHAR); } +"ENC_DB" { PUSH_STATE(ENC_DB); } +"ENC_LOCAL" { PUSH_STATE(ENC_LOCAL); } +"map.float" { PUSH_STATE(TSDB_FLOAT); } +"map.bigint" { PUSH_STATE(TSDB_BIGINT); } +[[:alnum:]_]+ { PUSH_STATE(KEY); } +.|\n { FAIL(); } -[[:space:]]+ { } -[=] { CHG_STATE(EQ); } -.|\n { return -1; } +[[:space:]]+ { } +[=] { PUSH_STATE(EQ); } +.|\n { FAIL(); } [[:space:]]+ { } -[^][{}(),;?*=!@/\\\n[:space:]]+ { set_val(); CHG_STATE(VAL); } [{] { CHG_STATE(BRACE1); } -.|\n { return -1; } +[^][{}(),;?*=!@/\\\n[:space:]]+ { set_val(); POP_STATE(); CHG_STATE(VAL); } +.|\n { FAIL(); } [^{}\n]+ { set_val(); CHG_STATE(BRACE2); } -.|\n { return -1; } +.|\n { FAIL(); } [[:space:]]+ { } -[}] { CHG_STATE(VAL); } -.|\n { return -1; } +[}] { POP_STATE(); CHG_STATE(VAL); } +.|\n { FAIL(); } [;] { POP_STATE(); } -.|\n { return -1; } +.|\n { FAIL(); } %% +static char* get_val_by_key_from_odbc_ini(const char *dsn, const char *key); + +static void conn_val_init(conn_val_t *val); + int todbc_parse_conn_string(const char *conn, conn_val_t *val) { yyscan_t arg = {0}; yylex_init(&arg); yyset_debug(0, arg); yyset_extra(val, arg); + + conn_val_init(val); + yy_scan_string(conn, arg); int ret =yylex(arg); yylex_destroy(arg); - if (val->key) free(val->key); val->key = NULL; - if (ret) { + if (ret || !val->dsn) { conn_val_reset(val); + } else { + if (!val->uid) { + val->uid = get_val_by_key_from_odbc_ini(val->dsn, "UID"); + } + if (!val->pwd) { + val->pwd = get_val_by_key_from_odbc_ini(val->dsn, "PWD"); + } + if (!val->server) { + val->server = get_val_by_key_from_odbc_ini(val->dsn, "Server"); + } } return ret ? -1 : 0; } void conn_val_reset(conn_val_t *val) { - if (val->key) { - free(val->key); val->key = NULL; - } if (val->dsn) { free(val->dsn); val->dsn = NULL; } @@ -155,11 +213,68 @@ void conn_val_reset(conn_val_t *val) { if (val->server) { free(val->server); val->server = NULL; } - if (val->svr_enc) { - free(val->svr_enc); val->svr_enc = NULL; + if (val->enc_local) { + free(val->enc_local); val->enc_local = NULL; + } + if (val->enc_db) { + free(val->enc_db); val->enc_db = NULL; + } + if (val->enc_char) { + free(val->enc_char); val->enc_char = NULL; } - if (val->cli_enc) { - free(val->cli_enc); val->cli_enc = NULL; + if (val->enc_wchar) { + free(val->enc_wchar); val->enc_wchar = NULL; + } +} + +static char* get_val_by_key_from_odbc_ini(const char *dsn, const char *key) { + char Val[4096]; + Val[0] = '\0'; + int n = SQLGetPrivateProfileString(dsn, key, "", Val, sizeof(Val), "odbc.ini"); + if (n<=0) return NULL; + if (Val[0]=='\0') return NULL; + return strdup(Val); +} + +static int process_map(const char *cfg, map_tsdb_type_t *tsdb_map, int type) { + switch (type) { + case TSDB_FLOAT: { + if (strcmp(cfg, "SQL_DOUBLE")==0) { + tsdb_map->tsdb_float = SQL_DOUBLE; + return 0; + } + } break; + case TSDB_BIGINT: { + if (strcmp(cfg, "SQL_C_SBIGINT")==0) { + tsdb_map->tsdb_bigint = SQL_C_SBIGINT; + return 0; + } + if (strcmp(cfg, "SQL_C_UBIGINT")==0) { + tsdb_map->tsdb_bigint = SQL_C_UBIGINT; + return 0; + } + if (strcmp(cfg, "SQL_CHAR")==0) { + tsdb_map->tsdb_bigint = SQL_CHAR; + return 0; + } + } break; + default: { + } break; } + return -1; +} + +static void conn_val_init(conn_val_t *val) { + if (!val) return; + val->tsdb_map.tsdb_tinyint = SQL_TINYINT; + val->tsdb_map.tsdb_smallint = SQL_SMALLINT; + val->tsdb_map.tsdb_int = SQL_INTEGER; + val->tsdb_map.tsdb_bigint = SQL_BIGINT; + val->tsdb_map.tsdb_float = SQL_REAL; + val->tsdb_map.tsdb_double = SQL_DOUBLE; + val->tsdb_map.tsdb_bool = SQL_TINYINT; + val->tsdb_map.tsdb_timestamp = SQL_CHAR; + val->tsdb_map.tsdb_binary = SQL_BINARY; + val->tsdb_map.tsdb_nchar = SQL_WCHAR; } diff --git a/src/connector/odbc/src/todbc_string.c b/src/connector/odbc/src/todbc_string.c new file mode 100644 index 0000000000000000000000000000000000000000..96e9a6d8c3658b08e9e57071aaf4d573e3ca81cb --- /dev/null +++ b/src/connector/odbc/src/todbc_string.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "todbc_string.h" + +#include "todbc_log.h" +#include "todbc_tls.h" + +#include + +static int do_calc_bytes(todbc_string_t *str); + +todbc_string_t todbc_string_init(const char *enc, const unsigned char *src, const size_t bytes) { + DASSERT(enc); + DASSERT(src); + + todbc_string_t s = {0}; + todbc_string_t *str = &s; + + todbc_iconv_t *cnv = todbc_tls_iconv_get(enc, UTF8_ENC); + if (!cnv) return s; + + if (snprintf(str->enc, sizeof(str->enc), "%s", enc)>=sizeof(str->enc)) { + return s; + } + str->buf = src; + str->total_bytes = bytes; // need to recalc + str->bytes = 0; + + if (do_calc_bytes(str)) { + str->buf = NULL; + str->total_bytes = 0; + str->bytes = 0; + } + + return s; +} + +todbc_string_t todbc_string_copy(todbc_string_t *str, const char *enc, unsigned char *dst, const size_t target_bytes) { + todbc_string_t val = {0}; + DASSERT(str); + DASSERT(dst); + DASSERT(str->buf); + DASSERT(str->bytes<=INT64_MAX); + DASSERT(str->total_bytes<=INT64_MAX); + + if (snprintf(val.enc, sizeof(val.enc), "%s", enc)>=sizeof(val.enc)) { + return val; + } + + todbc_iconv_t *icnv = todbc_tls_iconv_get(enc, str->enc); + if (!icnv) return val; + iconv_t cnv = todbc_iconv_get(icnv); + if (cnv==(iconv_t)-1) return val; + + val.buf = dst; + val.total_bytes = target_bytes; + + const int null_bytes = todbc_iconv_to(icnv).null_size; + + if (target_bytes<=null_bytes) return val; + size_t estsize = todbc_iconv_est_bytes(icnv, str->bytes); + if (estsize>INT64_MAX) return val; + + // smaller is better!!! + const size_t outblock = (estsize > target_bytes) ? estsize = target_bytes : estsize; + + char *inbuf = (char*)str->buf; + size_t inbytes = str->bytes; // not counting null-terminator + char *outbuf = (char*)dst; + size_t outbytes = outblock; + + int r = todbc_iconv_raw(icnv, (const unsigned char*)inbuf, &inbytes, (unsigned char*)outbuf, &outbytes); + if (r) { + DASSERT(outbytes > 0); + val.bytes = outblock - outbytes; + val.total_bytes = outblock; + return val; + } else { + val.bytes = outblock - outbytes; + val.total_bytes = val.bytes; + if (inbytes > 0) { + val.total_bytes += 1; // to indicate truncation + } + return val; + } +} + +todbc_string_t todbc_copy(const char *from_enc, const unsigned char *src, size_t *inbytes, const char *to_enc, unsigned char *dst, const size_t dlen) { + DASSERT(from_enc); + DASSERT(src); + DASSERT(inbytes); + DASSERT(to_enc); + DASSERT(dst); + DASSERT(dlen <= INT64_MAX); + + todbc_string_t s_from = todbc_string_init(from_enc, src, *inbytes); + DASSERT(s_from.buf == src); + + return todbc_string_copy(&s_from, to_enc, dst, dlen); +} + +todbc_string_t todbc_string_conv_to(todbc_string_t *str, const char *enc, todbc_buf_t *buf) { + DASSERT(str); + DASSERT(str->buf); + DASSERT(str->bytes<=INT64_MAX); + DASSERT(str->total_bytes<=INT64_MAX); + + todbc_string_t nul = {0}; + + todbc_iconv_t *icnv = todbc_tls_iconv_get(enc, str->enc); + if (!icnv) return nul; + + size_t estsize = todbc_iconv_est_bytes(icnv, str->bytes); + if (estsize>INT64_MAX) return nul; + char *out = NULL; + if (!buf) out = (char*)todbc_tls_buf_alloc(estsize); + else out = (char*)todbc_buf_alloc(buf, estsize); + if (!out) return nul; + + return todbc_string_copy(str, enc, (unsigned char*)out, estsize); +} + +static int do_calc_bytes(todbc_string_t *str) { + iconv_t cnv = todbc_tls_iconv(UTF8_ENC, str->enc); + if (cnv == (iconv_t)-1) return -1; + + size_t total_bytes = 0; + + char buf[1024*16]; + + char *inbuf = (char*)str->buf; + + while (1) { + size_t outblock = sizeof(buf); + + size_t inblock = outblock; + size_t remain = (size_t)-1; + if (str->total_bytes <= INT64_MAX) { + remain = str->total_bytes - total_bytes; + if (remain==0) break; + if (inblock > remain) inblock = remain; + } + + size_t inbytes = inblock; + char *outbuf = buf; + size_t outbytes = outblock; + + size_t n = iconv(cnv, &inbuf, &inbytes, &outbuf, &outbytes); + total_bytes += inblock - inbytes; + + int e = 0; + if (n==(size_t)-1) { + e = errno; + if (str->total_bytes<=INT64_MAX) { + D("iconv failed @[%zu], inbytes[%zd->%zd], outbytes[%zd->%zd]: [%d]%s", + (inblock-inbytes), inblock, inbytes, outblock, outbytes, e, strerror(e)); + } + DASSERT(e==EILSEQ || e==E2BIG || e==EINVAL); + } + if (n>0 && n<=INT64_MAX) { + D("iconv found non-reversible seq"); + } + + size_t outlen = outblock - outbytes; + size_t utf8len = strnlen(buf, outlen); + if (utf8len < outlen) { + // null-terminator found + // revert + inbuf -= inblock - inbytes; + total_bytes -= inblock - inbytes; + + if (utf8len==0) break; + + inbytes = inblock; + outbuf = buf; + outbytes = utf8len; + + n = iconv(cnv, &inbuf, &inbytes, &outbuf, &outbytes); + total_bytes += inblock - inbytes; + DASSERT(n==(size_t)-1); + e = errno; + DASSERT(e==E2BIG); + + break; + } + + if (e==EILSEQ) break; + if (e==EINVAL) { + if (inbytes == remain) { + // this is the last stuff + break; + } + } + } + + if (str->total_bytes > INT64_MAX) { + str->total_bytes = total_bytes; + } + str->bytes = total_bytes; + + iconv(cnv, NULL, NULL, NULL, NULL); + + return 0; +} + diff --git a/src/connector/odbc/src/todbc_string.h b/src/connector/odbc/src/todbc_string.h new file mode 100644 index 0000000000000000000000000000000000000000..eed3356847a16ab383da63eaa8c503312273136a --- /dev/null +++ b/src/connector/odbc/src/todbc_string.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _todbc_string_h_ +#define _todbc_string_h_ + +#include +#include + +#include "todbc_buf.h" + +// non-thread-safe + +typedef struct todbc_string_s todbc_string_t; +struct todbc_string_s { + // null if init failed because of internal resources shortage + const unsigned char *buf; // null-terminator inclusive + size_t total_bytes; // not counting null-terminator + + // <= total_bytes + // truncated if < total_bytes + size_t bytes; // not counting null-terminator + + // move here to satisfy todbc_string_t dummy = {0}; + char enc[64]; +}; + + +// does not copy internally +// bytes: not characters, <0 means bytes unknown +todbc_string_t todbc_string_init(const char *enc, const unsigned char *src, const size_t bytes); +// conv and copy to dst not more than target_bytes (null-terminator-inclusive) +// return'd val->buf == dst, total_bytes + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "todbc_tls.h" + +#include "todbc_buf.h" +#include "todbc_iconv.h" +#include "todbc_log.h" + + +typedef struct todbc_tls_s todbc_tls_t; + +struct todbc_tls_s { + todbc_buf_t *buf; + todbc_iconvset_t *cnvset; +}; + + +static void todbc_tls_free(todbc_tls_t *value); + +static pthread_key_t key_this; +static pthread_once_t key_once = PTHREAD_ONCE_INIT; +static int key_err = 0; + + +static void key_init(void); +static void key_destructor(void *arg); + + +static void key_init(void) { + key_err = pthread_key_create(&key_this, key_destructor); + if (key_err) { + D("thread local initialization failed: [%d]%s", key_err, strerror(key_err)); + } +} + +static todbc_tls_t* todbc_tls_create(void); + +static todbc_tls_t* key_value(void) { + pthread_once(&key_once, key_init); + if (key_err) return NULL; + + int err = 0; + + todbc_tls_t *value = pthread_getspecific(key_this); + if (value) return value; + + value = todbc_tls_create(); + if (!value) return NULL; + + do { + err = pthread_setspecific(key_this, value); + if (err) { + D("thread local setup failed: [%d]%s", err, strerror(err)); + break; + } + + return value; + } while (0); + + todbc_tls_free(value); + + return NULL; +} + +static void key_destructor(void *arg) { + todbc_tls_t *value = (todbc_tls_t*)arg; + todbc_tls_free(value); +} + +static todbc_tls_t* todbc_tls_create(void) { + int err = 0; + todbc_tls_t *value = (todbc_tls_t*)calloc(1, sizeof(*value)); + if (!value) { + err = errno; + D("thread local creation failed: [%d]%s", err, strerror(err)); + return NULL; + } + do { + return value; + } while (0); + + todbc_tls_free(value); + return NULL; +} + +static void todbc_tls_free(todbc_tls_t *value) { + if (value->cnvset) { + todbc_iconvset_free(value->cnvset); + value->cnvset = NULL; + } + + if (value->buf) { + todbc_buf_free(value->buf); + value->buf = NULL; + } + + free(value); +} + +static todbc_iconvset_t* do_get_iconvset(void); + +// iconv +int todbc_legal_chars(const char *enc, const unsigned char *str, todbc_bytes_t *bc) { + todbc_iconvset_t *icnv = do_get_iconvset(); + if (!icnv) return -1; + todbc_iconv_t *cnv = todbc_iconvset_get(icnv, UTF16_ENC, enc); + if (!cnv) return -1; + return todbc_iconv_get_legal_chars(cnv, str, bc); +} + +todbc_iconv_t* todbc_tls_iconv_get(const char *to_enc, const char *from_enc) { + todbc_iconvset_t *cnvset = do_get_iconvset(); + if (!cnvset) return NULL; + todbc_iconv_t *cnv = todbc_iconvset_get(cnvset, to_enc, from_enc); + return cnv; +} + +iconv_t todbc_tls_iconv(const char *to_enc, const char *from_enc) { + todbc_iconv_t *icnv = todbc_tls_iconv_get(to_enc, from_enc); + if (!icnv) return (iconv_t)-1; + return todbc_iconv_get(icnv); +} + +todbc_enc_t todbc_tls_iconv_enc(const char *enc) { + do { + todbc_iconvset_t *cnvset = do_get_iconvset(); + if (!cnvset) break; + return todbc_iconvset_enc(cnvset, enc); + } while (0); + + todbc_enc_t v = {0}; + v.char_size = -1; + v.null_size = -1; + + return v; +} + +todbc_string_t todbc_tls_conv(todbc_buf_t *buf, const char *enc_to, const char *enc_from, const unsigned char *src, size_t *slen) { + todbc_iconv_t *cnv = todbc_tls_iconv_get(enc_to, enc_from); + if (!cnv) { + todbc_string_t nul = {0}; + return nul; + } + return todbc_iconv_conv2(cnv, buf, src, slen); +} + +todbc_string_t todbc_tls_write(const char *enc_to, const char *enc_from, + const unsigned char *src, size_t *slen, unsigned char *dst, size_t dlen) +{ + todbc_iconv_t *cnv = todbc_tls_iconv_get(enc_to, enc_from); + if (!cnv) { + todbc_string_t nul = {0}; + return nul; + } + todbc_string_t s = {0}; + s.buf = dst; + s.total_bytes = dlen; + size_t inbytes = *slen; + size_t outbytes = dlen; + todbc_iconv_raw(cnv, src, &inbytes, dst, &outbytes); + s.bytes = dlen - outbytes; + s.total_bytes = s.bytes; + if (inbytes) { + s.total_bytes += 1; + } + *slen = inbytes; + + return s; +} + +char* todbc_tls_strndup(const char *src, size_t n) { + todbc_buf_t *buf = todbc_tls_buf(); + if (!buf) return NULL; + n = strnlen(src, n); + char *d = todbc_buf_alloc(buf, (n+1)); + if (!d) return NULL; + snprintf(d, n+1, "%s", src); + return d; +} + +static todbc_iconvset_t* do_get_iconvset(void) { + todbc_tls_t *tls = key_value(); + if (!tls) return NULL; + if (!tls->cnvset) { + tls->cnvset = todbc_iconvset_create(); + } + return tls->cnvset; +} + +// tls_buf +void* todbc_tls_buf_alloc(size_t size) { + todbc_tls_t *tls = key_value(); + if (!tls) return NULL; + if (!tls->buf) { + tls->buf = todbc_buf_create(); + if (!tls->buf) return NULL; + } + return todbc_buf_alloc(tls->buf, size); +} + +void* todbc_tls_buf_calloc(size_t count, size_t size) { + todbc_tls_t *tls = key_value(); + if (!tls) return NULL; + if (!tls->buf) { + tls->buf = todbc_buf_create(); + if (!tls->buf) return NULL; + } + return todbc_buf_calloc(tls->buf, count, size); +} + +void* todbc_tls_buf_realloc(void *ptr, size_t size) { + todbc_tls_t *tls = key_value(); + if (!tls) return NULL; + if (!tls->buf) { + tls->buf = todbc_buf_create(); + if (!tls->buf) return NULL; + } + return todbc_buf_realloc(tls->buf, ptr, size); +} + +void todbc_tls_buf_reclaim(void) { + todbc_tls_t *tls = key_value(); + if (!tls) return; + if (!tls->buf) return; + + todbc_buf_reclaim(tls->buf); +} + +todbc_buf_t* todbc_tls_buf(void) { + todbc_tls_t *tls = key_value(); + if (!tls) return NULL; + if (!tls->buf) { + tls->buf = todbc_buf_create(); + } + return tls->buf; +} + diff --git a/src/connector/odbc/src/todbc_tls.h b/src/connector/odbc/src/todbc_tls.h new file mode 100644 index 0000000000000000000000000000000000000000..e636f6ae6c9c9051fe4a77d5e9d1e082e2dcfdb0 --- /dev/null +++ b/src/connector/odbc/src/todbc_tls.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _todbc_tls_h_ +#define _todbc_tls_h_ + +// !!! functions exported in this header file are all non-thread-safe !!! + +#include "taos.h" + +#include "todbc_buf.h" +#include "todbc_iconv.h" +#include "todbc_string.h" + +// thread local buffers +// non-thread-safe +// returned-buf are all thread-local-accessible until todbc_tls_buf_reclaim +void* todbc_tls_buf_alloc(size_t size); +void* todbc_tls_buf_calloc(size_t count, size_t size); +void* todbc_tls_buf_realloc(void *ptr, size_t size); +// reclaim all above thread-local-buf(s) +void todbc_tls_buf_reclaim(void); + +// return local-thread-buf +todbc_buf_t* todbc_tls_buf(void); + +// thread local iconv +// non-thread-safe +todbc_iconv_t* todbc_tls_iconv_get(const char *to_enc, const char *from_enc); +iconv_t todbc_tls_iconv(const char *to_enc, const char *from_enc); + +todbc_enc_t todbc_tls_iconv_enc(const char *enc); + +// non-thread-safe +int todbc_legal_chars(const char *enc, const unsigned char *str, todbc_bytes_t *bc); + +// at return, *slen stores the remaining # +todbc_string_t todbc_tls_conv(todbc_buf_t *buf, const char *enc_to, const char *enc_from, const unsigned char *src, size_t *slen); +todbc_string_t todbc_tls_write(const char *enc_to, const char *enc_from, const unsigned char *src, size_t *slen, unsigned char *dst, size_t dlen); + +char* todbc_tls_strndup(const char *src, size_t n); + +#endif // _todbc_tls_h_ + diff --git a/src/connector/odbc/src/todbc_util.c b/src/connector/odbc/src/todbc_util.c index 9c130b4f2f2e6d0f5d4a19e11ab323d42b800195..d0b5abe09321c0b65b711b4e45e23e038c3dff88 100644 --- a/src/connector/odbc/src/todbc_util.c +++ b/src/connector/odbc/src/todbc_util.c @@ -15,86 +15,576 @@ #include "todbc_util.h" #include "todbc_log.h" + #include #include +#define SQL_CASE(type) case type: return #type + const char* sql_sql_type(int type) { switch (type) { - case SQL_BIT: return "SQL_BIT"; - case SQL_TINYINT: return "SQL_TINYINT"; - case SQL_SMALLINT: return "SQL_SMALLINT"; - case SQL_INTEGER: return "SQL_INTEGER"; - case SQL_BIGINT: return "SQL_BIGINT"; - case SQL_FLOAT: return "SQL_FLOAT"; - case SQL_DOUBLE: return "SQL_DOUBLE"; - case SQL_DECIMAL: return "SQL_DECIMAL"; - case SQL_NUMERIC: return "SQL_NUMERIC"; - case SQL_REAL: return "SQL_REAL"; - case SQL_CHAR: return "SQL_CHAR"; - case SQL_VARCHAR: return "SQL_VARCHAR"; - case SQL_LONGVARCHAR: return "SQL_LONGVARCHAR"; - case SQL_WCHAR: return "SQL_WCHAR"; - case SQL_WVARCHAR: return "SQL_WVARCHAR"; - case SQL_WLONGVARCHAR: return "SQL_WLONGVARCHAR"; - case SQL_BINARY: return "SQL_BINARY"; - case SQL_VARBINARY: return "SQL_VARBINARY"; - case SQL_LONGVARBINARY: return "SQL_LONGVARBINARY"; - case SQL_DATE: return "SQL_DATE"; - case SQL_TIME: return "SQL_TIME"; - case SQL_TIMESTAMP: return "SQL_TIMESTAMP"; - case SQL_TYPE_DATE: return "SQL_TYPE_DATE"; - case SQL_TYPE_TIME: return "SQL_TYPE_TIME"; - case SQL_TYPE_TIMESTAMP: return "SQL_TYPE_TIMESTAMP"; - case SQL_INTERVAL_MONTH: return "SQL_INTERVAL_MONTH"; - case SQL_INTERVAL_YEAR: return "SQL_INTERVAL_YEAR"; - case SQL_INTERVAL_YEAR_TO_MONTH: return "SQL_INTERVAL_YEAR_TO_MONTH"; - case SQL_INTERVAL_DAY: return "SQL_INTERVAL_DAY"; - case SQL_INTERVAL_HOUR: return "SQL_INTERVAL_HOUR"; - case SQL_INTERVAL_MINUTE: return "SQL_INTERVAL_MINUTE"; - case SQL_INTERVAL_SECOND: return "SQL_INTERVAL_SECOND"; - case SQL_INTERVAL_DAY_TO_HOUR: return "SQL_INTERVAL_DAY_TO_HOUR"; - case SQL_INTERVAL_DAY_TO_MINUTE: return "SQL_INTERVAL_DAY_TO_MINUTE"; - case SQL_INTERVAL_DAY_TO_SECOND: return "SQL_INTERVAL_DAY_TO_SECOND"; - case SQL_INTERVAL_HOUR_TO_MINUTE: return "SQL_INTERVAL_HOUR_TO_MINUTE"; - case SQL_INTERVAL_HOUR_TO_SECOND: return "SQL_INTERVAL_HOUR_TO_SECOND"; - case SQL_INTERVAL_MINUTE_TO_SECOND: return "SQL_INTERVAL_MINUTE_TO_SECOND"; - case SQL_GUID: return "SQL_GUID"; + SQL_CASE(SQL_BIT); + SQL_CASE(SQL_TINYINT); + SQL_CASE(SQL_SMALLINT); + SQL_CASE(SQL_INTEGER); + SQL_CASE(SQL_BIGINT); + SQL_CASE(SQL_FLOAT); + SQL_CASE(SQL_DOUBLE); + SQL_CASE(SQL_DECIMAL); + SQL_CASE(SQL_NUMERIC); + SQL_CASE(SQL_REAL); + SQL_CASE(SQL_CHAR); + SQL_CASE(SQL_VARCHAR); + SQL_CASE(SQL_LONGVARCHAR); + SQL_CASE(SQL_WCHAR); + SQL_CASE(SQL_WVARCHAR); + SQL_CASE(SQL_WLONGVARCHAR); + SQL_CASE(SQL_BINARY); + SQL_CASE(SQL_VARBINARY); + SQL_CASE(SQL_LONGVARBINARY); + SQL_CASE(SQL_DATE); + SQL_CASE(SQL_TIME); + SQL_CASE(SQL_TIMESTAMP); + SQL_CASE(SQL_TYPE_DATE); + SQL_CASE(SQL_TYPE_TIME); + SQL_CASE(SQL_TYPE_TIMESTAMP); + SQL_CASE(SQL_INTERVAL_MONTH); + SQL_CASE(SQL_INTERVAL_YEAR); + SQL_CASE(SQL_INTERVAL_YEAR_TO_MONTH); + SQL_CASE(SQL_INTERVAL_DAY); + SQL_CASE(SQL_INTERVAL_HOUR); + SQL_CASE(SQL_INTERVAL_MINUTE); + SQL_CASE(SQL_INTERVAL_SECOND); + SQL_CASE(SQL_INTERVAL_DAY_TO_HOUR); + SQL_CASE(SQL_INTERVAL_DAY_TO_MINUTE); + SQL_CASE(SQL_INTERVAL_DAY_TO_SECOND); + SQL_CASE(SQL_INTERVAL_HOUR_TO_MINUTE); + SQL_CASE(SQL_INTERVAL_HOUR_TO_SECOND); + SQL_CASE(SQL_INTERVAL_MINUTE_TO_SECOND); + SQL_CASE(SQL_GUID); + SQL_CASE(SQL_ALL_TYPES); default: return "UNKNOWN"; } } const char* sql_c_type(int type) { switch (type) { - case SQL_C_CHAR: return "SQL_C_CHAR"; - case SQL_C_WCHAR: return "SQL_C_WCHAR"; - case SQL_C_SHORT: return "SQL_C_SHORT"; - case SQL_C_SSHORT: return "SQL_C_SSHORT"; - case SQL_C_USHORT: return "SQL_C_USHORT"; - case SQL_C_LONG: return "SQL_C_LONG"; - case SQL_C_SLONG: return "SQL_C_SLONG"; - case SQL_C_ULONG: return "SQL_C_ULONG"; - case SQL_C_FLOAT: return "SQL_C_FLOAT"; - case SQL_C_DOUBLE: return "SQL_C_DOUBLE"; - case SQL_C_BIT: return "SQL_C_BIT"; - case SQL_C_TINYINT: return "SQL_C_TINYINT"; - case SQL_C_STINYINT: return "SQL_C_STINYINT"; - case SQL_C_UTINYINT: return "SQL_C_UTINYINT"; - case SQL_C_SBIGINT: return "SQL_C_SBIGINT"; - case SQL_C_UBIGINT: return "SQL_C_UBIGINT"; - case SQL_C_BINARY: return "SQL_C_BINARY"; - case SQL_C_DATE: return "SQL_C_DATE"; - case SQL_C_TIME: return "SQL_C_TIME"; - case SQL_C_TIMESTAMP: return "SQL_C_TIMESTAMP"; - case SQL_C_TYPE_DATE: return "SQL_C_TYPE_DATE"; - case SQL_C_TYPE_TIME: return "SQL_C_TYPE_TIME"; - case SQL_C_TYPE_TIMESTAMP: return "SQL_C_TYPE_TIMESTAMP"; - case SQL_C_NUMERIC: return "SQL_C_NUMERIC"; - case SQL_C_GUID: return "SQL_C_GUID"; + SQL_CASE(SQL_C_CHAR); + SQL_CASE(SQL_C_WCHAR); + SQL_CASE(SQL_C_SHORT); + SQL_CASE(SQL_C_SSHORT); + SQL_CASE(SQL_C_USHORT); + SQL_CASE(SQL_C_LONG); + SQL_CASE(SQL_C_SLONG); + SQL_CASE(SQL_C_ULONG); + SQL_CASE(SQL_C_FLOAT); + SQL_CASE(SQL_C_DOUBLE); + SQL_CASE(SQL_C_BIT); + SQL_CASE(SQL_C_TINYINT); + SQL_CASE(SQL_C_STINYINT); + SQL_CASE(SQL_C_UTINYINT); + SQL_CASE(SQL_C_SBIGINT); + SQL_CASE(SQL_C_UBIGINT); + SQL_CASE(SQL_C_BINARY); + SQL_CASE(SQL_C_DATE); + SQL_CASE(SQL_C_TIME); + SQL_CASE(SQL_C_TIMESTAMP); + SQL_CASE(SQL_C_TYPE_DATE); + SQL_CASE(SQL_C_TYPE_TIME); + SQL_CASE(SQL_C_TYPE_TIMESTAMP); + SQL_CASE(SQL_C_NUMERIC); + SQL_CASE(SQL_C_GUID); + default: return "UNKNOWN"; + } +} + +// https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlgetdiagfield-function?view=sql-server-ver15 +const char* sql_diag_identifier(int type) { + switch (type) { + // header fields + SQL_CASE(SQL_DIAG_CURSOR_ROW_COUNT); + SQL_CASE(SQL_DIAG_DYNAMIC_FUNCTION); + SQL_CASE(SQL_DIAG_DYNAMIC_FUNCTION_CODE); + SQL_CASE(SQL_DIAG_NUMBER); + SQL_CASE(SQL_DIAG_RETURNCODE); + SQL_CASE(SQL_DIAG_ROW_COUNT); + // record fields + SQL_CASE(SQL_DIAG_CLASS_ORIGIN); + SQL_CASE(SQL_DIAG_COLUMN_NUMBER); + SQL_CASE(SQL_DIAG_CONNECTION_NAME); + SQL_CASE(SQL_DIAG_MESSAGE_TEXT); + SQL_CASE(SQL_DIAG_NATIVE); + SQL_CASE(SQL_DIAG_ROW_NUMBER); + SQL_CASE(SQL_DIAG_SERVER_NAME); + SQL_CASE(SQL_DIAG_SQLSTATE); + SQL_CASE(SQL_DIAG_SUBCLASS_ORIGIN); + default: return "UNKNOWN"; + } +} + +const char* sql_handle_type(int type) { + switch(type) { + SQL_CASE(SQL_HANDLE_ENV); + SQL_CASE(SQL_HANDLE_DBC); + SQL_CASE(SQL_HANDLE_STMT); + SQL_CASE(SQL_HANDLE_DESC); + // SQL_CASE(SQL_HANDLE_DBC_INFO_TOKEN); + default: return "UNKNOWN"; + } +} + +const char* sql_env_attr_type(int type) { + switch(type) { + SQL_CASE(SQL_ATTR_OUTPUT_NTS); + SQL_CASE(SQL_ATTR_ODBC_VERSION); + SQL_CASE(SQL_ATTR_CONNECTION_POOLING); + SQL_CASE(SQL_ATTR_CP_MATCH); + default: return "UNKNOWN"; + } +} + +// https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlsetconnectattr-function?view=sql-server-ver15 +const char* sql_conn_attr_type(int type) { + switch(type) { + SQL_CASE(SQL_ATTR_ACCESS_MODE); + // ODBC 3.8 + // SQL_CASE(SQL_ATTR_ASYNC_DBC_EVENT); + SQL_CASE(SQL_ATTR_ASYNC_DBC_FUNCTIONS_ENABLE); + // ODBC 3.8 + // SQL_CASE(SQL_ATTR_ASYNC_DBC_PCALLBACK); + // ODBC 3.8 + // SQL_CASE(SQL_ATTR_ASYNC_DBC_PCONTEXT); + SQL_CASE(SQL_ATTR_ASYNC_ENABLE); + SQL_CASE(SQL_ATTR_AUTO_IPD); + SQL_CASE(SQL_ATTR_AUTOCOMMIT); + SQL_CASE(SQL_ATTR_CONNECTION_DEAD); + SQL_CASE(SQL_ATTR_CONNECTION_TIMEOUT); + SQL_CASE(SQL_ATTR_CURRENT_CATALOG); + // ODBC 3.8 + // SQL_CASE(SQL_ATTR_DBC_INFO_TOKEN); + SQL_CASE(SQL_ATTR_ENLIST_IN_DTC); + SQL_CASE(SQL_ATTR_LOGIN_TIMEOUT); + SQL_CASE(SQL_ATTR_METADATA_ID); + SQL_CASE(SQL_ATTR_ODBC_CURSORS); + SQL_CASE(SQL_ATTR_PACKET_SIZE); + SQL_CASE(SQL_ATTR_QUIET_MODE); + SQL_CASE(SQL_ATTR_TRACE); + SQL_CASE(SQL_ATTR_TRACEFILE); + SQL_CASE(SQL_ATTR_TRANSLATE_LIB); + SQL_CASE(SQL_ATTR_TRANSLATE_OPTION); + SQL_CASE(SQL_ATTR_TXN_ISOLATION); + default: return "UNKNOWN"; + } +} + +// https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlgetinfo-function?view=sql-server-ver15 +const char* sql_info_type(int type) { + switch(type) { + SQL_CASE(SQL_ACTIVE_ENVIRONMENTS); + SQL_CASE(SQL_ASYNC_DBC_FUNCTIONS); + SQL_CASE(SQL_ASYNC_MODE); + SQL_CASE(SQL_ASYNC_NOTIFICATION); + SQL_CASE(SQL_BATCH_ROW_COUNT); + SQL_CASE(SQL_BATCH_SUPPORT); + SQL_CASE(SQL_DATA_SOURCE_NAME); + SQL_CASE(SQL_DRIVER_AWARE_POOLING_SUPPORTED); + SQL_CASE(SQL_DRIVER_HDBC); + SQL_CASE(SQL_DRIVER_HDESC); + SQL_CASE(SQL_DRIVER_HENV); + SQL_CASE(SQL_DRIVER_HLIB); + SQL_CASE(SQL_DRIVER_HSTMT); + SQL_CASE(SQL_DRIVER_NAME); + SQL_CASE(SQL_DRIVER_ODBC_VER); + SQL_CASE(SQL_DRIVER_VER); + SQL_CASE(SQL_DYNAMIC_CURSOR_ATTRIBUTES1); + SQL_CASE(SQL_DYNAMIC_CURSOR_ATTRIBUTES2); + SQL_CASE(SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES1); + SQL_CASE(SQL_FORWARD_ONLY_CURSOR_ATTRIBUTES2); + SQL_CASE(SQL_FILE_USAGE); + SQL_CASE(SQL_GETDATA_EXTENSIONS); + SQL_CASE(SQL_INFO_SCHEMA_VIEWS); + SQL_CASE(SQL_KEYSET_CURSOR_ATTRIBUTES1); + SQL_CASE(SQL_KEYSET_CURSOR_ATTRIBUTES2); + SQL_CASE(SQL_MAX_ASYNC_CONCURRENT_STATEMENTS); + SQL_CASE(SQL_MAX_CONCURRENT_ACTIVITIES); + SQL_CASE(SQL_MAX_DRIVER_CONNECTIONS); + SQL_CASE(SQL_ODBC_INTERFACE_CONFORMANCE); + // SQL_CASE(SQL_ODBC_STANDARD_CLI_CONFORMANCE); + SQL_CASE(SQL_ODBC_VER); + SQL_CASE(SQL_PARAM_ARRAY_ROW_COUNTS); + SQL_CASE(SQL_PARAM_ARRAY_SELECTS); + SQL_CASE(SQL_ROW_UPDATES); + SQL_CASE(SQL_SEARCH_PATTERN_ESCAPE); + SQL_CASE(SQL_SERVER_NAME); + SQL_CASE(SQL_STATIC_CURSOR_ATTRIBUTES1); + SQL_CASE(SQL_STATIC_CURSOR_ATTRIBUTES2); + + SQL_CASE(SQL_DATABASE_NAME); + SQL_CASE(SQL_DBMS_NAME); + SQL_CASE(SQL_DBMS_VER); + + SQL_CASE(SQL_ACCESSIBLE_PROCEDURES); + SQL_CASE(SQL_ACCESSIBLE_TABLES); + SQL_CASE(SQL_BOOKMARK_PERSISTENCE); + SQL_CASE(SQL_CATALOG_TERM); + SQL_CASE(SQL_COLLATION_SEQ); + SQL_CASE(SQL_CONCAT_NULL_BEHAVIOR); + SQL_CASE(SQL_CURSOR_COMMIT_BEHAVIOR); + SQL_CASE(SQL_CURSOR_ROLLBACK_BEHAVIOR); + SQL_CASE(SQL_CURSOR_SENSITIVITY); + SQL_CASE(SQL_DATA_SOURCE_READ_ONLY); + SQL_CASE(SQL_DEFAULT_TXN_ISOLATION); + SQL_CASE(SQL_DESCRIBE_PARAMETER); + SQL_CASE(SQL_MULT_RESULT_SETS); + SQL_CASE(SQL_MULTIPLE_ACTIVE_TXN); + SQL_CASE(SQL_NEED_LONG_DATA_LEN); + SQL_CASE(SQL_NULL_COLLATION); + SQL_CASE(SQL_PROCEDURE_TERM); + SQL_CASE(SQL_SCHEMA_TERM); + SQL_CASE(SQL_SCROLL_OPTIONS); + SQL_CASE(SQL_TABLE_TERM); + SQL_CASE(SQL_TXN_CAPABLE); + SQL_CASE(SQL_TXN_ISOLATION_OPTION); + SQL_CASE(SQL_USER_NAME); + + SQL_CASE(SQL_AGGREGATE_FUNCTIONS); + SQL_CASE(SQL_ALTER_DOMAIN); + // SQL_CASE(SQL_ALTER_SCHEMA); + SQL_CASE(SQL_ALTER_TABLE); + // SQL_CASE(SQL_ANSI_SQL_DATETIME_LITERALS); + SQL_CASE(SQL_CATALOG_LOCATION); + SQL_CASE(SQL_CATALOG_NAME); + SQL_CASE(SQL_CATALOG_NAME_SEPARATOR); + SQL_CASE(SQL_CATALOG_USAGE); + SQL_CASE(SQL_COLUMN_ALIAS); + SQL_CASE(SQL_CORRELATION_NAME); + SQL_CASE(SQL_CREATE_ASSERTION); + SQL_CASE(SQL_CREATE_CHARACTER_SET); + SQL_CASE(SQL_CREATE_COLLATION); + SQL_CASE(SQL_CREATE_DOMAIN); + SQL_CASE(SQL_CREATE_SCHEMA); + SQL_CASE(SQL_CREATE_TABLE); + SQL_CASE(SQL_CREATE_TRANSLATION); + SQL_CASE(SQL_DDL_INDEX); + SQL_CASE(SQL_DROP_ASSERTION); + SQL_CASE(SQL_DROP_CHARACTER_SET); + SQL_CASE(SQL_DROP_COLLATION); + SQL_CASE(SQL_DROP_DOMAIN); + SQL_CASE(SQL_DROP_SCHEMA); + + SQL_CASE(SQL_DROP_TABLE); + SQL_CASE(SQL_DROP_TRANSLATION); + SQL_CASE(SQL_DROP_VIEW); + SQL_CASE(SQL_EXPRESSIONS_IN_ORDERBY); + SQL_CASE(SQL_GROUP_BY); + SQL_CASE(SQL_IDENTIFIER_CASE); + SQL_CASE(SQL_IDENTIFIER_QUOTE_CHAR); + SQL_CASE(SQL_INDEX_KEYWORDS); + SQL_CASE(SQL_INSERT_STATEMENT); + SQL_CASE(SQL_INTEGRITY); + SQL_CASE(SQL_KEYWORDS); + SQL_CASE(SQL_LIKE_ESCAPE_CLAUSE); + SQL_CASE(SQL_NON_NULLABLE_COLUMNS); + SQL_CASE(SQL_OJ_CAPABILITIES); + SQL_CASE(SQL_ORDER_BY_COLUMNS_IN_SELECT); + SQL_CASE(SQL_OUTER_JOINS); + SQL_CASE(SQL_PROCEDURES); + SQL_CASE(SQL_QUOTED_IDENTIFIER_CASE); + SQL_CASE(SQL_SCHEMA_USAGE); + SQL_CASE(SQL_SPECIAL_CHARACTERS); + SQL_CASE(SQL_SQL_CONFORMANCE); + SQL_CASE(SQL_SUBQUERIES); + SQL_CASE(SQL_UNION); + + SQL_CASE(SQL_MAX_BINARY_LITERAL_LEN); + SQL_CASE(SQL_MAX_CATALOG_NAME_LEN); + SQL_CASE(SQL_MAX_CHAR_LITERAL_LEN); + SQL_CASE(SQL_MAX_COLUMN_NAME_LEN); + SQL_CASE(SQL_MAX_COLUMNS_IN_GROUP_BY); + SQL_CASE(SQL_MAX_COLUMNS_IN_INDEX); + SQL_CASE(SQL_MAX_COLUMNS_IN_ORDER_BY); + SQL_CASE(SQL_MAX_COLUMNS_IN_SELECT); + SQL_CASE(SQL_MAX_COLUMNS_IN_TABLE); + SQL_CASE(SQL_MAX_CURSOR_NAME_LEN); + + SQL_CASE(SQL_MAX_IDENTIFIER_LEN); + SQL_CASE(SQL_MAX_INDEX_SIZE); + SQL_CASE(SQL_MAX_PROCEDURE_NAME_LEN); + SQL_CASE(SQL_MAX_ROW_SIZE); + SQL_CASE(SQL_MAX_ROW_SIZE_INCLUDES_LONG); + SQL_CASE(SQL_MAX_SCHEMA_NAME_LEN); + SQL_CASE(SQL_MAX_STATEMENT_LEN); + SQL_CASE(SQL_MAX_TABLE_NAME_LEN); + SQL_CASE(SQL_MAX_TABLES_IN_SELECT); + SQL_CASE(SQL_MAX_USER_NAME_LEN); + + SQL_CASE(SQL_CONVERT_FUNCTIONS); + SQL_CASE(SQL_NUMERIC_FUNCTIONS); + SQL_CASE(SQL_STRING_FUNCTIONS); + SQL_CASE(SQL_SYSTEM_FUNCTIONS); + + SQL_CASE(SQL_TIMEDATE_ADD_INTERVALS); + SQL_CASE(SQL_TIMEDATE_DIFF_INTERVALS); + SQL_CASE(SQL_TIMEDATE_FUNCTIONS); + + SQL_CASE(SQL_CONVERT_BIGINT); + SQL_CASE(SQL_CONVERT_BINARY); + SQL_CASE(SQL_CONVERT_BIT); + SQL_CASE(SQL_CONVERT_CHAR); + SQL_CASE(SQL_CONVERT_DATE); + SQL_CASE(SQL_CONVERT_DECIMAL); + SQL_CASE(SQL_CONVERT_DOUBLE); + SQL_CASE(SQL_CONVERT_FLOAT); + SQL_CASE(SQL_CONVERT_INTEGER); + SQL_CASE(SQL_CONVERT_INTERVAL_DAY_TIME); + SQL_CASE(SQL_CONVERT_INTERVAL_YEAR_MONTH); + + SQL_CASE(SQL_CONVERT_LONGVARBINARY); + SQL_CASE(SQL_CONVERT_LONGVARCHAR); + SQL_CASE(SQL_CONVERT_NUMERIC); + SQL_CASE(SQL_CONVERT_REAL); + SQL_CASE(SQL_CONVERT_SMALLINT); + SQL_CASE(SQL_CONVERT_TIME); + SQL_CASE(SQL_CONVERT_TIMESTAMP); + SQL_CASE(SQL_CONVERT_TINYINT); + SQL_CASE(SQL_CONVERT_VARBINARY); + SQL_CASE(SQL_CONVERT_VARCHAR); + + SQL_CASE(SQL_DM_VER); + + SQL_CASE(SQL_XOPEN_CLI_YEAR); + + SQL_CASE(SQL_DTC_TRANSITION_COST); + + default: return "UNKNOWN"; + } +} + +// https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlcolattribute-function?view=sql-server-ver15 +const char* sql_field_identifier(int type) { + switch (type) { + SQL_CASE(SQL_DESC_AUTO_UNIQUE_VALUE); + SQL_CASE(SQL_DESC_BASE_COLUMN_NAME); + SQL_CASE(SQL_DESC_BASE_TABLE_NAME); + SQL_CASE(SQL_DESC_CASE_SENSITIVE); + SQL_CASE(SQL_DESC_CATALOG_NAME); + SQL_CASE(SQL_DESC_CONCISE_TYPE); + SQL_CASE(SQL_DESC_COUNT); + SQL_CASE(SQL_DESC_DISPLAY_SIZE); + SQL_CASE(SQL_DESC_FIXED_PREC_SCALE); + SQL_CASE(SQL_DESC_LABEL); + SQL_CASE(SQL_DESC_LENGTH); + SQL_CASE(SQL_DESC_LITERAL_PREFIX); + SQL_CASE(SQL_DESC_LITERAL_SUFFIX); + SQL_CASE(SQL_DESC_LOCAL_TYPE_NAME); + SQL_CASE(SQL_DESC_NAME); + SQL_CASE(SQL_DESC_NULLABLE); + SQL_CASE(SQL_DESC_NUM_PREC_RADIX); + SQL_CASE(SQL_DESC_OCTET_LENGTH); + SQL_CASE(SQL_DESC_PRECISION); + SQL_CASE(SQL_DESC_SCALE); + SQL_CASE(SQL_DESC_SCHEMA_NAME); + SQL_CASE(SQL_DESC_SEARCHABLE); + SQL_CASE(SQL_DESC_TABLE_NAME); + SQL_CASE(SQL_DESC_TYPE); + SQL_CASE(SQL_DESC_TYPE_NAME); + SQL_CASE(SQL_DESC_UNNAMED); + SQL_CASE(SQL_DESC_UNSIGNED); + SQL_CASE(SQL_DESC_UPDATABLE); default: return "UNKNOWN"; } } +// https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlbindparameter-function?view=sql-server-ver15 +const char* sql_input_output_type(int type) { + switch (type) { + SQL_CASE(SQL_PARAM_INPUT); + SQL_CASE(SQL_PARAM_OUTPUT); + SQL_CASE(SQL_PARAM_OUTPUT_STREAM); + SQL_CASE(SQL_PARAM_INPUT_OUTPUT); + SQL_CASE(SQL_PARAM_INPUT_OUTPUT_STREAM); + default: return "UNKNOWN"; + } +} + +// https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqlsetstmtattr-function?view=sql-server-ver15 +const char* sql_stmt_attr_type(int type) { + switch (type) { + SQL_CASE(SQL_ATTR_APP_PARAM_DESC); + SQL_CASE(SQL_ATTR_APP_ROW_DESC); + SQL_CASE(SQL_ATTR_ASYNC_ENABLE); + SQL_CASE(SQL_ATTR_ASYNC_STMT_EVENT); + // ODBC 3.8 + // SQL_CASE(SQL_ATTR_ASYNC_STMT_PCALLBACK); + // ODBC 3.8 + // SQL_CASE(SQL_ATTR_ASYNC_STMT_PCONTEXT); + SQL_CASE(SQL_ATTR_CONCURRENCY); + SQL_CASE(SQL_ATTR_CURSOR_SCROLLABLE); + SQL_CASE(SQL_ATTR_CURSOR_SENSITIVITY); + SQL_CASE(SQL_ATTR_CURSOR_TYPE); + SQL_CASE(SQL_ATTR_ENABLE_AUTO_IPD); + SQL_CASE(SQL_ATTR_FETCH_BOOKMARK_PTR); + SQL_CASE(SQL_ATTR_IMP_PARAM_DESC); + SQL_CASE(SQL_ATTR_IMP_ROW_DESC); + SQL_CASE(SQL_ATTR_KEYSET_SIZE); + SQL_CASE(SQL_ATTR_MAX_LENGTH); + SQL_CASE(SQL_ATTR_MAX_ROWS); + SQL_CASE(SQL_ATTR_METADATA_ID); + SQL_CASE(SQL_ATTR_NOSCAN); + SQL_CASE(SQL_ATTR_PARAM_BIND_OFFSET_PTR); + SQL_CASE(SQL_ATTR_PARAM_BIND_TYPE); + SQL_CASE(SQL_ATTR_PARAM_OPERATION_PTR); + SQL_CASE(SQL_ATTR_PARAM_STATUS_PTR); + SQL_CASE(SQL_ATTR_PARAMS_PROCESSED_PTR); + SQL_CASE(SQL_ATTR_PARAMSET_SIZE); + SQL_CASE(SQL_ATTR_QUERY_TIMEOUT); + SQL_CASE(SQL_ATTR_RETRIEVE_DATA); + SQL_CASE(SQL_ATTR_ROW_ARRAY_SIZE); + SQL_CASE(SQL_ATTR_ROW_BIND_OFFSET_PTR); + SQL_CASE(SQL_ATTR_ROW_BIND_TYPE); + SQL_CASE(SQL_ATTR_ROW_NUMBER); + SQL_CASE(SQL_ATTR_ROW_OPERATION_PTR); + SQL_CASE(SQL_ATTR_ROW_STATUS_PTR); + SQL_CASE(SQL_ATTR_ROWS_FETCHED_PTR); + SQL_CASE(SQL_ATTR_SIMULATE_CURSOR); + SQL_CASE(SQL_ATTR_USE_BOOKMARKS); + default: return "UNKNOWN"; + } +} + +const char* sql_function_type(int type) { + switch (type) { + // + SQL_CASE(SQL_API_ALL_FUNCTIONS); + SQL_CASE(SQL_API_ODBC3_ALL_FUNCTIONS); + + // ISO 92 standards-compliance + SQL_CASE(SQL_API_SQLALLOCHANDLE); + SQL_CASE(SQL_API_SQLGETDESCFIELD); + SQL_CASE(SQL_API_SQLBINDCOL); + SQL_CASE(SQL_API_SQLGETDESCREC); + SQL_CASE(SQL_API_SQLCANCEL); + SQL_CASE(SQL_API_SQLGETDIAGFIELD); + SQL_CASE(SQL_API_SQLCLOSECURSOR); + SQL_CASE(SQL_API_SQLGETDIAGREC); + SQL_CASE(SQL_API_SQLCOLATTRIBUTE); + SQL_CASE(SQL_API_SQLGETENVATTR); + SQL_CASE(SQL_API_SQLCONNECT); + SQL_CASE(SQL_API_SQLGETFUNCTIONS); + SQL_CASE(SQL_API_SQLCOPYDESC); + SQL_CASE(SQL_API_SQLGETINFO); + SQL_CASE(SQL_API_SQLDATASOURCES); + SQL_CASE(SQL_API_SQLGETSTMTATTR); + SQL_CASE(SQL_API_SQLDESCRIBECOL); + SQL_CASE(SQL_API_SQLGETTYPEINFO); + SQL_CASE(SQL_API_SQLDISCONNECT); + SQL_CASE(SQL_API_SQLNUMRESULTCOLS); + SQL_CASE(SQL_API_SQLDRIVERS); + SQL_CASE(SQL_API_SQLPARAMDATA); + SQL_CASE(SQL_API_SQLENDTRAN); + SQL_CASE(SQL_API_SQLPREPARE); + SQL_CASE(SQL_API_SQLEXECDIRECT); + SQL_CASE(SQL_API_SQLPUTDATA); + SQL_CASE(SQL_API_SQLEXECUTE); + SQL_CASE(SQL_API_SQLROWCOUNT); + SQL_CASE(SQL_API_SQLFETCH); + SQL_CASE(SQL_API_SQLSETCONNECTATTR); + SQL_CASE(SQL_API_SQLFETCHSCROLL); + SQL_CASE(SQL_API_SQLSETCURSORNAME); + SQL_CASE(SQL_API_SQLFREEHANDLE); + SQL_CASE(SQL_API_SQLSETDESCFIELD); + SQL_CASE(SQL_API_SQLFREESTMT); + SQL_CASE(SQL_API_SQLSETDESCREC); + SQL_CASE(SQL_API_SQLGETCONNECTATTR); + SQL_CASE(SQL_API_SQLSETENVATTR); + SQL_CASE(SQL_API_SQLGETCURSORNAME); + SQL_CASE(SQL_API_SQLSETSTMTATTR); + SQL_CASE(SQL_API_SQLGETDATA); + + // Open Group standards-compliance); + SQL_CASE(SQL_API_SQLCOLUMNS); + SQL_CASE(SQL_API_SQLSTATISTICS); + SQL_CASE(SQL_API_SQLSPECIALCOLUMNS); + SQL_CASE(SQL_API_SQLTABLES); + + // ODBC standards-compliance); + SQL_CASE(SQL_API_SQLBINDPARAMETER); + SQL_CASE(SQL_API_SQLNATIVESQL); + SQL_CASE(SQL_API_SQLBROWSECONNECT); + SQL_CASE(SQL_API_SQLNUMPARAMS); + SQL_CASE(SQL_API_SQLBULKOPERATIONS); + SQL_CASE(SQL_API_SQLPRIMARYKEYS); + SQL_CASE(SQL_API_SQLCOLUMNPRIVILEGES); + SQL_CASE(SQL_API_SQLPROCEDURECOLUMNS); + SQL_CASE(SQL_API_SQLDESCRIBEPARAM); + SQL_CASE(SQL_API_SQLPROCEDURES); + SQL_CASE(SQL_API_SQLDRIVERCONNECT); + SQL_CASE(SQL_API_SQLSETPOS); + SQL_CASE(SQL_API_SQLFOREIGNKEYS); + SQL_CASE(SQL_API_SQLTABLEPRIVILEGES); + SQL_CASE(SQL_API_SQLMORERESULTS); + + SQL_CASE(SQL_API_SQLALLOCCONNECT); + SQL_CASE(SQL_API_SQLALLOCENV); + SQL_CASE(SQL_API_SQLALLOCSTMT); + SQL_CASE(SQL_API_SQLBINDPARAM); + SQL_CASE(SQL_API_SQLERROR); + SQL_CASE(SQL_API_SQLFREECONNECT); + SQL_CASE(SQL_API_SQLFREEENV); + SQL_CASE(SQL_API_SQLGETCONNECTOPTION); + SQL_CASE(SQL_API_SQLGETSTMTOPTION); + SQL_CASE(SQL_API_SQLSETCONNECTOPTION); + SQL_CASE(SQL_API_SQLSETPARAM); + SQL_CASE(SQL_API_SQLSETSTMTOPTION); + SQL_CASE(SQL_API_SQLTRANSACT); + SQL_CASE(SQL_API_SQLCANCELHANDLE); + + default: return "UNKNOWN"; + } +} + +const char* sql_freestmt_option_type(int type) { + switch (type) { + SQL_CASE(SQL_CLOSE); + SQL_CASE(SQL_DROP); + SQL_CASE(SQL_UNBIND); + SQL_CASE(SQL_RESET_PARAMS); + default: return "UNKNOWN"; + } +} + +const char* sql_soi_type(int soi) { + switch (soi) { + SQL_CASE(SQL_NTS); + SQL_CASE(SQL_NULL_DATA); + SQL_CASE(SQL_DEFAULT_PARAM); + SQL_CASE(SQL_DATA_AT_EXEC); + default: { + if (soi >= 0) return ""; + return "SQL_LEN_DATA_AT_EXEC(?)"; + } break; + } +} + +const char* sql_nullable_type(int type) { + switch (type) { + SQL_CASE(SQL_NO_NULLS); + SQL_CASE(SQL_NULLABLE); + SQL_CASE(SQL_NULLABLE_UNKNOWN); + default: { + return "UNKNOWN"; + } break; + } +} + + + + + + + + int is_valid_sql_c_type(int type) { const char *ctype = sql_c_type(type); if (strcmp(ctype, "UNKNOWN")==0) return 0; @@ -127,3 +617,46 @@ int utf8_chars(const char *src) return (int)chars; } +static int do_charset_chars(iconv_t cnv, const unsigned char *src) +{ + int chars = 0; + char *ps = (char*)src; + char buf[16]; + size_t sn = 1; + while (1) { + char *ds = buf; + size_t dn = sizeof(buf); + size_t n = iconv(cnv, &ps, &sn, &ds, &dn); + if (n==(size_t)-1) { + int e = errno; + switch (e) { + case EILSEQ: return -1; + case E2BIG: return -1; + case EINVAL: sn += 1; continue; + default: return -1; + } + } + if (sn) return -1; + if (n>0) return -1; + int i=0; + for (i=0; i<(sizeof(buf)-dn); ++i) { + if (buf[i]) break; + } + if (i>=(sizeof(buf)-dn)) break; + chars += (int)1; + sn = 1; + } + return chars; +} + +int charset_chars(const char *charset, const unsigned char *src) { + iconv_t cnv = iconv_open(charset, charset); + if (cnv==(iconv_t)-1) return -1; + + int chars = do_charset_chars(cnv, src); + + iconv_close(cnv); + + return chars; +} + diff --git a/src/connector/odbc/src/todbc_util.h b/src/connector/odbc/src/todbc_util.h index ead0d7348973409c85741cc4d676e40f6f140447..a340c32546e0b40ba05eda190c03944d1cea4d55 100644 --- a/src/connector/odbc/src/todbc_util.h +++ b/src/connector/odbc/src/todbc_util.h @@ -23,11 +23,24 @@ const char* sql_sql_type(int type); const char* sql_c_type(int type); +const char* sql_handle_type(int type); +const char* sql_env_attr_type(int type); +const char* sql_conn_attr_type(int type); +const char* sql_info_type(int type); +const char* sql_field_identifier(int type); +const char* sql_diag_identifier(int type); +const char* sql_input_output_type(int type); +const char* sql_stmt_attr_type(int type); +const char* sql_function_type(int type); +const char* sql_freestmt_option_type(int type); +const char* sql_soi_type(int soi); +const char* sql_nullable_type(int type); int is_valid_sql_c_type(int type); int is_valid_sql_sql_type(int type); int utf8_chars(const char *src); +int charset_chars(const char *charset, const unsigned char *src); #endif // _TODBC_UTIL_H_ diff --git a/src/connector/odbc/tests/CMakeLists.txt b/src/connector/odbc/tests/CMakeLists.txt deleted file mode 100644 index 1cc6acaf4bf34aa2158cc1f4fa0836d6e51f3a41..0000000000000000000000000000000000000000 --- a/src/connector/odbc/tests/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -PROJECT(TDengine) - -IF (TD_LINUX) - # AUX_SOURCE_DIRECTORY(. SRC) - ADD_EXECUTABLE(tcodbc main.c) - TARGET_LINK_LIBRARIES(tcodbc odbc) - ADD_EXECUTABLE(tconv tconv.c) -ENDIF () - -IF (TD_WINDOWS_64) - SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /GL") - SET(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /GL") - # AUX_SOURCE_DIRECTORY(. SRC) - ADD_EXECUTABLE(tcodbc main.c) - TARGET_LINK_LIBRARIES(tcodbc odbc32 odbccp32 user32 legacy_stdio_definitions os) - ADD_EXECUTABLE(tconv tconv.c) - TARGET_LINK_LIBRARIES(tconv tutil) -ENDIF () diff --git a/src/connector/odbc/tests/main.c b/src/connector/odbc/tests/main.c deleted file mode 100644 index 417de00d55f64249a9194b77fecbeb458c560cc7..0000000000000000000000000000000000000000 --- a/src/connector/odbc/tests/main.c +++ /dev/null @@ -1,673 +0,0 @@ -#include "../src/todbc_log.h" - -#ifdef _MSC_VER -#include -#include -#include "os.h" -#endif -#include -#include -#include - -#include -#include - -#define CHK_TEST(statement) \ -do { \ - D("testing: %s", #statement); \ - int r = (statement); \ - if (r) { \ - D("testing failed: %s", #statement); \ - return 1; \ - } \ -} while (0); - -typedef struct db_column_s db_column_t; -struct db_column_s { - SQLSMALLINT nameLength; - char name[4096]; // seems enough - SQLSMALLINT dataType; - SQLULEN columnSize; - SQLSMALLINT decimalDigits; - SQLSMALLINT nullable; -}; - -static db_column_t *columns = NULL; - -typedef struct data_s data_t; -struct data_s { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40+1]; - char blob[40+1]; // why 80? ref: tests/examples/c/apitest.c -}; - -static const char *pre_stmts[] = { - "create database db", - "use db", - "create table t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))" -}; - -static const char *pro_stmts[] = { - // "insert into t values ('2019-07-15 00:00:00', 1)", - // "insert into t values ('2019-07-15 01:00:00', 2)", - "select * from t" - // "drop database db" -}; - -#define CHK_RESULT(r, ht, h, fmt, ...) \ -do { \ - if (r==0) break; \ - SQLCHAR ss[10]; \ - SQLINTEGER ne = 0; \ - SQLCHAR es[4096]; \ - SQLSMALLINT n = 0; \ - ss[0] = '\0'; \ - es[0] = '\0'; \ - SQLRETURN ret = SQLGetDiagRec(ht, h, 1, ss, &ne, es, sizeof(es), &n); \ - if (ret) break; \ - D("[%s]%s: " fmt "", ss, es, ##__VA_ARGS__); \ -} while (0) - -static int open_connect(const char *dsn, const char *uid, const char *pwd, SQLHENV *pEnv, SQLHDBC *pConn) { - SQLRETURN r; - SQLHENV env = {0}; - SQLHDBC conn = {0}; - r = SQLAllocEnv(&env); - if (r!=SQL_SUCCESS) return 1; - do { - r = SQLAllocConnect(env, &conn); - CHK_RESULT(r, SQL_HANDLE_ENV, env, ""); - if (r!=SQL_SUCCESS) break; - do { - r = SQLConnect(conn, (SQLCHAR*)dsn, (SQLSMALLINT)(dsn ? strlen(dsn) : 0), - (SQLCHAR*)uid, (SQLSMALLINT)(uid ? strlen(uid) : 0), - (SQLCHAR*)pwd, (SQLSMALLINT)(pwd ? strlen(pwd) : 0)); - CHK_RESULT(r, SQL_HANDLE_DBC, conn, ""); - if (r==SQL_SUCCESS) { - *pEnv = env; - *pConn = conn; - return 0; - } - } while (0); - SQLFreeConnect(conn); - } while (0); - SQLFreeEnv(env); - - return 1; -} - -static int open_driver_connect(const char *connstr, SQLHENV *pEnv, SQLHDBC *pConn) { - SQLRETURN r; - SQLHENV env = {0}; - SQLHDBC conn = {0}; - r = SQLAllocEnv(&env); - if (r!=SQL_SUCCESS) return 1; - do { - r = SQLAllocConnect(env, &conn); - CHK_RESULT(r, SQL_HANDLE_ENV, env, ""); - if (r!=SQL_SUCCESS) break; - do { - SQLCHAR buf[4096]; - SQLSMALLINT blen = 0; - SQLHDBC ConnectionHandle = conn; - SQLHWND WindowHandle = NULL; - SQLCHAR * InConnectionString = (SQLCHAR*)connstr; - SQLSMALLINT StringLength1 = (SQLSMALLINT)(connstr ? strlen(connstr) : 0); - SQLCHAR * OutConnectionString = buf; - SQLSMALLINT BufferLength = sizeof(buf); - SQLSMALLINT * StringLength2Ptr = &blen; - SQLUSMALLINT DriverCompletion = SQL_DRIVER_NOPROMPT; - r = SQLDriverConnect(ConnectionHandle, WindowHandle, InConnectionString, - StringLength1, OutConnectionString, BufferLength, - StringLength2Ptr, DriverCompletion); - CHK_RESULT(r, SQL_HANDLE_DBC, conn, ""); - if (r==SQL_SUCCESS) { - *pEnv = env; - *pConn = conn; - return 0; - } - } while (0); - SQLFreeConnect(conn); - } while (0); - SQLFreeEnv(env); - - return 1; -} - -static SQLRETURN traverse_cols(SQLHSTMT stmt, SQLSMALLINT cols) { - SQLRETURN r = SQL_ERROR; - for (SQLSMALLINT i=0; i0) fprintf(stdout, "\n"); - return r; - } - } - if (soi==SQL_NULL_DATA) { - fprintf(stdout, "%snull", i==0?"":","); - } else { - fprintf(stdout, "%s\"%s\"", i==0?"":",", buf); - } - } - fprintf(stdout, "\n"); - } - - // r = SQLFetch(stmt); - // if (r==SQL_NO_DATA) { - // D(".........."); - // r = SQL_SUCCESS; - // break; - // } - // CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); - // if (r) break; - // r = SQLPrepare(stmt, (SQLCHAR*)statement, strlen(statement)); - // CHK_RESULT(r, SQL_HANDLE_STMT, stmt, ""); - // if (r) break; - // r = SQLExecute(stmt); - // CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - // if (r) break; - } while (0); - return r; -} - -static int do_insert(SQLHSTMT stmt, data_t data) { - SQLRETURN r = 0; - SQLLEN lbin; - SQLLEN lblob; - - const char *statement = "insert into t values (?, ?, ?, ?, ?, ?, ?, ?, ?,?)"; - #define ignored 0 - - do { - r = SQLPrepare(stmt, (SQLCHAR*)statement, (SQLINTEGER)strlen(statement)); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLBindParameter(stmt, 1, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_TIMESTAMP, ignored, ignored, &data.ts, ignored, NULL); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLBindParameter(stmt, 2, SQL_PARAM_INPUT, SQL_C_BIT, SQL_BIT, ignored, ignored, &data.b, ignored, NULL); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLBindParameter(stmt, 3, SQL_PARAM_INPUT, SQL_C_TINYINT, SQL_TINYINT, ignored, ignored, &data.v1, ignored, NULL); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLBindParameter(stmt, 4, SQL_PARAM_INPUT, SQL_C_SHORT, SQL_SMALLINT, ignored, ignored, &data.v2, ignored, NULL); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLBindParameter(stmt, 5, SQL_PARAM_INPUT, SQL_C_LONG, SQL_INTEGER, ignored, ignored, &data.v4, ignored, NULL); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLBindParameter(stmt, 6, SQL_PARAM_INPUT, SQL_C_SBIGINT, SQL_BIGINT, ignored, ignored, &data.v8, ignored, NULL); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLBindParameter(stmt, 7, SQL_PARAM_INPUT, SQL_C_FLOAT, SQL_FLOAT, ignored, ignored, &data.f4, ignored, NULL); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLBindParameter(stmt, 8, SQL_PARAM_INPUT, SQL_C_DOUBLE, SQL_DOUBLE, ignored, ignored, &data.f8, ignored, NULL); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - lbin = SQL_NTS; - r = SQLBindParameter(stmt, 9, SQL_PARAM_INPUT, SQL_C_BINARY, SQL_VARBINARY, sizeof(data.bin)-1, ignored, &data.bin, ignored, &lbin); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - lblob = SQL_NTS; - r = SQLBindParameter(stmt, 10, SQL_PARAM_INPUT, SQL_C_CHAR, SQL_VARCHAR, sizeof(data.blob)-1, ignored, &data.blob, ignored, &lblob); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - r = SQLExecute(stmt); - CHK_RESULT(r, SQL_HANDLE_STMT, stmt, "statement: %s", statement); - if (r) break; - - // ts += 1; - // v = 2; - // r = SQLExecute(stmt); - // if (r) break; - } while (0); - - #undef ignored - return r; -} - -static int test1(const char *dsn, const char *uid, const char *pwd) { - SQLHENV env = {0}; - SQLHDBC conn = {0}; - int n = open_connect(dsn, uid, pwd, &env, &conn); - if (n) return 1; - - int ok = 0; - do { - SQLRETURN r = SQL_SUCCESS; - SQLHSTMT stmt = {0}; - r = SQLAllocHandle(SQL_HANDLE_STMT, conn, &stmt); - if (r!=SQL_SUCCESS) break; - do { - if (do_statement(stmt, "drop database if exists db")) { - break; - } - for (size_t i=0; i0 && line[n-1] == '\n') line[n-1]='\0'; - if (n>0 && line[n-1] == '\r') line[n-1]='\0'; - if (n>1 && line[n-2] == '\r') line[n-2]='\0'; - p = line; - while (isspace(*p)) ++p; - - if (*p==0) break; - - int positive = 1; - if (strncmp(p, "N:", 2)==0) { - positive = 0; - p += 2; - } else if (strncmp(p, "P:", 2)==0) { - p += 2; - } - - D("statement: [%s]", p); - r = do_statement(stmt, p); - - if (positive && r==0) break; - if (!positive && r) { r = 0; break; } - if (positive) return r; - D("expecting negative result, but got positive"); - return -1; - } while (0); - - free(line); - - if (r) break; - } - - fclose(f); - return r ? 1 : 0; -} - -int test_sqls_in_conn(SQLHENV env, SQLHDBC conn, const char *sqls) { - SQLHSTMT stmt = {0}; - CHK_TEST(create_statement(env, conn, &stmt)); - int r = test_sqls_in_stmt(env, conn, stmt, sqls); - SQLFreeHandle(SQL_HANDLE_STMT, stmt); - return r ? 1 : 0; -} - -int test_sqls(const char *dsn, const char *uid, const char *pwd, const char *connstr, const char *sqls) { - int r = 0; - SQLHENV env = {0}; - SQLHDBC conn = {0}; - if (dsn) { - CHK_TEST(open_connect(dsn, uid, pwd, &env, &conn)); - } else { - CHK_TEST(open_driver_connect(connstr, &env, &conn)); - } - if (sqls) { - r = test_sqls_in_conn(env, conn, sqls); - } - SQLDisconnect(conn); - SQLFreeConnect(conn); - SQLFreeEnv(env); - return r ? 1 : 0; -} - -void usage(const char *arg0) { - fprintf(stdout, "%s usage:\n", arg0); - fprintf(stdout, "%s [--dsn ] [--uid ] [--pwd ] [--dcs ] [--sts ]\n", arg0); - fprintf(stdout, " --dsn : DSN\n"); - fprintf(stdout, " --uid : UID\n"); - fprintf(stdout, " --pwd : PWD\n"); - fprintf(stdout, " --dcs : driver connection string\n"); - fprintf(stdout, " --sts : file where statements store\n"); -} - -int main(int argc, char *argv[]) { - // if (argc==1) { - // CHK_TEST(test_env()); - // CHK_TEST(test1("TAOS_DSN", "root", "taoxsdata")); - // D("Done!"); - // return 0; - // } - - const char *dsn = NULL; - const char *uid = NULL; - const char *pwd = NULL; - const char *dcs = NULL; // driver connection string - const char *sts = NULL; // statements file - for (size_t i=1; i=argc) { - D(" expected but got nothing"); - return 1; - } - if (dcs) { - D("--dcs has already been specified"); - return 1; - } - dsn = argv[i]; - continue; - } - if (strcmp(arg, "--uid")==0) { - ++i; - if (i>=argc) { - D(" expected but got nothing"); - return 1; - } - uid = argv[i]; - continue; - } - if (strcmp(arg, "--pwd")==0) { - ++i; - if (i>=argc) { - D(" expected but got nothing"); - return 1; - } - pwd = argv[i]; - continue; - } - if (strcmp(arg, "--dcs")==0) { - ++i; - if (i>=argc) { - D(" expected but got nothing"); - return 1; - } - if (dsn || uid || pwd) { - D("either of --dsn/--uid/--pwd has already been specified"); - return 1; - } - dcs = argv[i]; - continue; - } - if (strcmp(arg, "--sts")==0) { - ++i; - if (i>=argc) { - D(" expected but got nothing"); - return 1; - } - sts = argv[i]; - continue; - } - } - CHK_TEST(test_sqls(dsn, uid, pwd, dcs, sts)); - D("Done!"); - return 0; - - if (0) { - const char *dsn = (argc>1) ? argv[1] : NULL; - const char *uid = (argc>2) ? argv[2] : NULL; - const char *pwd = (argc>3) ? argv[3] : NULL; - const char *connstr = (argc>4) ? argv[4] : NULL; - const char *sqls = (argc>5) ? argv[5] : NULL; - - dsn = NULL; - uid = NULL; - pwd = NULL; - connstr = argv[1]; - sqls = argv[2]; - if (0) { - CHK_TEST(test_env()); - - CHK_TEST(test1(dsn, uid, pwd)); - - const char *statements[] = { - "drop database if exists m", - "create database m", - "use m", - "drop database m", - NULL - }; - CHK_TEST(test_statements(dsn, uid, pwd, statements)); - - if (connstr) - CHK_TEST(test_driver_connect(connstr)); - - if (connstr) { - SQLHENV env = {0}; - SQLHDBC conn = {0}; - CHK_TEST(open_driver_connect(connstr, &env, &conn)); - int r = tests(env, conn); - SQLDisconnect(conn); - SQLFreeConnect(conn); - SQLFreeEnv(env); - if (r) return 1; - } - } - - if ((dsn || connstr) && 1) { - CHK_TEST(test_sqls(dsn, uid, pwd, connstr, sqls)); - } - - D("Done!"); - return 0; - } -} - diff --git a/src/connector/odbc/tests/odbc.py b/src/connector/odbc/tests/odbc.py deleted file mode 100644 index c137905775e567f6163846690886850cb77a684a..0000000000000000000000000000000000000000 --- a/src/connector/odbc/tests/odbc.py +++ /dev/null @@ -1,131 +0,0 @@ -import pyodbc -# cnxn = pyodbc.connect('DSN={TAOS_DSN};UID={ root };PWD={ taosdata };HOST={ localhost:6030 }', autocommit=True) -cnxn = pyodbc.connect('DSN={TAOS_DSN}; UID=root;PWD=taosdata; HOST=localhost:6030', autocommit=True) -cnxn.setdecoding(pyodbc.SQL_CHAR, encoding='utf-8') -#cnxn.setdecoding(pyodbc.SQL_WCHAR, encoding='utf-8') -#cnxn.setencoding(encoding='utf-8') - -#cursor = cnxn.cursor() -#cursor.execute("SELECT * from db.t") -#row = cursor.fetchone() -#while row: -# print(row) -# row = cursor.fetchone() -#cursor.close() - -#cursor = cnxn.cursor() -#cursor.execute(""" -#INSERT INTO db.t values (?,?,?,?,?,?,?,?,?,?) -#""", -#"2020-12-12 00:00:00", -#1, -#27, -#32767, -#147483647, -#223372036854775807, -#23.456, -#899.999999, -#"foo", -#"bar") - -cursor = cnxn.cursor() -cursor.execute("drop database if exists db"); -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("create database db"); -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("create table db.mt (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))"); -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("insert into db.mt values('2020-10-13 06:44:00', 1, 127, 32767, 32768, 32769, 123.456, 789.987, 'hello', 'world')") -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("insert into db.mt values(?,?,?,?,?,?,?,?,?,?)", "2020-10-13 07:06:00", 0, 127, 32767, 32768, 32769, 123.456, 789.987, "hel后lo", "wo哈rld"); -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("SELECT * from db.mt") -row = cursor.fetchone() -while row: - print(row) - row = cursor.fetchone() -cursor.close() - -#cursor = cnxn.cursor() -#cursor.execute("drop database if exists db"); -#cursor.close() -# -#cursor = cnxn.cursor() -#cursor.execute("create database db"); -#cursor.close() - -cursor = cnxn.cursor() -cursor.execute("create table db.t (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(4), blob nchar(4))"); -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("insert into db.t values('2020-10-13 06:44:00', 1, 127, 32767, 32768, 32769, 123.456, 789.987, 'hell', 'worl')") -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("SELECT * from db.t") -row = cursor.fetchone() -while row: - print(row) - row = cursor.fetchone() -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("create table db.v (ts timestamp, v1 tinyint)") -cursor.close() - -params = [ ('A', 1), ('B', 2), ('C', 3) ] -params = [ ('A', 1), ('B', 2), ('C', 3) ] -params = [ ('2020-10-16 00:00:00', 1), - ('2020-10-16 00:00:01', 4), - ('2020-10-16 00:00:02', 5), - ('2020-10-16 00:00:03.009', 6) ] -cursor = cnxn.cursor() -cursor.fast_executemany = True -cursor.executemany("insert into db.v values (?, ?)", params) -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("SELECT * from db.v") -row = cursor.fetchone() -while row: - print(row) - row = cursor.fetchone() -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("SELECT * from db.v where v1 > ?", 4) -row = cursor.fetchone() -while row: - print(row) - row = cursor.fetchone() -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("SELECT * from db.v where v1 > ?", '5') -row = cursor.fetchone() -while row: - print(row) - row = cursor.fetchone() -cursor.close() - -cursor = cnxn.cursor() -cursor.execute("create table db.f (ts timestamp, v1 float)") -cursor.close() - -params = [ ('2020-10-20 00:00:10', '123.3') ] -cursor = cnxn.cursor() -cursor.fast_executemany = True -cursor.executemany("insert into db.f values (?, ?)", params) -cursor.close() - diff --git a/src/connector/odbc/tools/CMakeLists.txt b/src/connector/odbc/tools/CMakeLists.txt index a0aafb1f3ca36bb0b6fb108e1948214b2dd14a6f..e543d245c8c3ef19f541832b3f2c8889860db2f9 100644 --- a/src/connector/odbc/tools/CMakeLists.txt +++ b/src/connector/odbc/tools/CMakeLists.txt @@ -1,12 +1,23 @@ PROJECT(TDengine) -IF (TD_LINUX) - ADD_EXECUTABLE(todbcinst main.c) +ADD_EXECUTABLE(todbcinst main.c) +ADD_EXECUTABLE(tconv tconv.c) + +IF (TD_LINUX OR TD_DARWIN) TARGET_LINK_LIBRARIES(todbcinst odbc odbcinst) ENDIF () +IF (TD_DARWIN) + target_include_directories(todbcinst PRIVATE /usr/local/include) + target_link_directories(todbcinst PUBLIC /usr/local/lib) + target_include_directories(tconv PRIVATE /usr/local/include) + target_link_directories(tconv PUBLIC /usr/local/lib) + TARGET_LINK_LIBRARIES(tconv iconv) +ENDIF () + IF (TD_WINDOWS_64) - ADD_EXECUTABLE(todbcinst main.c) TARGET_LINK_LIBRARIES(todbcinst odbc32 odbccp32 user32 legacy_stdio_definitions os) + TARGET_LINK_LIBRARIES(tconv taos) INSTALL(FILES ${EXECUTABLE_OUTPUT_PATH}/todbcinst.exe DESTINATION .) ENDIF () + diff --git a/src/connector/odbc/tests/tconv.c b/src/connector/odbc/tools/tconv.c similarity index 100% rename from src/connector/odbc/tests/tconv.c rename to src/connector/odbc/tools/tconv.c diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index 92a931b504fd76d12c1a4094a04fed4f859c0a3e..d1fca047c67861706e6e6afc4bc05cd888bff755 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.3", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/__init__.py b/src/connector/python/linux/python2/taos/__init__.py index 62e0536b6fca63a0aca0b6be3673104c27c631c6..973263573808232e4e71dc0158585624a8e7d2ab 100644 --- a/src/connector/python/linux/python2/taos/__init__.py +++ b/src/connector/python/linux/python2/taos/__init__.py @@ -3,12 +3,12 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' __all__ = ['connection', 'cursor'] + def connect(*args, **kwargs): """ Function to return a TDengine connector object @@ -21,4 +21,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) \ No newline at end of file + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 2b1b29ee31af56499d626fc56d4950e2e25f9978..4367947341edad7b0a9bdbcaec69a7de9801e267 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -4,11 +4,14 @@ from .error import * import math import datetime + def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli/1000.0) + return datetime.datetime.fromtimestamp(milli / 1000.0) + def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro/1000000.0) + return datetime.datetime.fromtimestamp(micro / 1000000.0) + def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row @@ -18,168 +21,309 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) + def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] - + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] + def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append( tmpstr.value.decode() ) + res.append(tmpstr.value.decode()) else: - res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) - return res + return res + def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows > 0: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) return res + def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows >= 0: for i in range(abs(num_of_rows)): try: - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode() ) + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res + _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } _CONVERT_FUNC_BLOCK = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python_block + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } # Corresponding TAOS_FIELD structure in C + + class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), ('type', ctypes.c_char), ('bytes', ctypes.c_short)] # C interface class + + class CTaosInterface(object): libtaos = ctypes.CDLL('libtaos.so') @@ -216,7 +360,7 @@ class CTaosInterface(object): except AttributeError: raise AttributeError("config is expected as a str") - if config != None: + if config is not None: CTaosInterface.libtaos.taos_options(3, self._config) CTaosInterface.libtaos.taos_init() @@ -227,7 +371,13 @@ class CTaosInterface(object): """ return self._config - def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): ''' Function to connect to server @@ -236,7 +386,7 @@ class CTaosInterface(object): # host try: _host = ctypes.c_char_p(host.encode( - "utf-8")) if host != None else ctypes.c_char_p(None) + "utf-8")) if host is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("host is expected as a str") @@ -255,7 +405,7 @@ class CTaosInterface(object): # db try: _db = ctypes.c_char_p( - db.encode("utf-8")) if db != None else ctypes.c_char_p(None) + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("db is expected as a str") @@ -268,11 +418,11 @@ class CTaosInterface(object): connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( _host, _user, _password, _db, _port)) - if connection.value == None: + if connection.value is None: print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - #else: + # else: # print('connect to TDengine success') return connection @@ -293,12 +443,13 @@ class CTaosInterface(object): @rtype: 0 on success and -1 on failure ''' try: - return CTaosInterface.libtaos.taos_query(connection, ctypes.c_char_p(sql.encode('utf-8'))) + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) except AttributeError: raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -308,7 +459,7 @@ class CTaosInterface(object): @staticmethod def subscribe(connection, restart, topic, sql, interval): """Create a subscription - @restart boolean, + @restart boolean, @sql string, sql statement for data query, must be a 'select' statement. @topic string, name of this subscription """ @@ -360,38 +511,53 @@ class CTaosInterface(object): result, ctypes.byref(pblock)) if num_of_rows == 0: return None, 0 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) + @staticmethod def fetchRow(result, fields): pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock : + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: num_of_rows = 1 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError("Invalid data type returned from database") + raise DatabaseError( + "Invalid data type returned from database") if data is None: blocks[i] = [None] else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) else: return None, 0 return blocks, abs(num_of_rows) + @staticmethod def freeResult(result): CTaosInterface.libtaos.taos_free_result(result) @@ -434,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -442,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') diff --git a/src/connector/python/linux/python2/taos/connection.py b/src/connector/python/linux/python2/taos/connection.py index 552250f1164ced467cd29b5084524985aca8848b..f6c395342c9c39a24bda6022f0ed36cb7bfe045b 100644 --- a/src/connector/python/linux/python2/taos/connection.py +++ b/src/connector/python/linux/python2/taos/connection.py @@ -2,9 +2,11 @@ from .cursor import TDengineCursor from .subscription import TDengineSubscription from .cinterface import CTaosInterface + class TDengineConnection(object): """ TDengine connection object """ + def __init__(self, *args, **kwargs): self._conn = None self._host = None @@ -29,7 +31,7 @@ class TDengineConnection(object): # password if 'password' in kwargs: self._password = kwargs['password'] - + # database if 'database' in kwargs: self._database = kwargs['database'] @@ -43,7 +45,12 @@ class TDengineConnection(object): self._config = kwargs['config'] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) def close(self): """Close current connection. @@ -55,7 +62,8 @@ class TDengineConnection(object): """ if self._conn is None: return None - sub = CTaosInterface.subscribe(self._conn, restart, topic, sql, interval) + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) return TDengineSubscription(sub) def cursor(self): @@ -80,7 +88,8 @@ class TDengineConnection(object): """ pass + if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') conn.close() - print("Hello world") \ No newline at end of file + print("Hello world") diff --git a/src/connector/python/linux/python2/taos/constants.py b/src/connector/python/linux/python2/taos/constants.py index feb7050a40b67f88a6d7ca859764fbbc6b36af1c..93466f5184a6bf37c2e1c915a00aa5c5e91d1801 100644 --- a/src/connector/python/linux/python2/taos/constants.py +++ b/src/connector/python/linux/python2/taos/constants.py @@ -3,6 +3,7 @@ from .dbapi import * + class FieldType(object): """TDengine Field Types """ @@ -18,13 +19,21 @@ class FieldType(object): C_BINARY = 8 C_TIMESTAMP = 9 C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index 82a01be67118813f9c83c3c3870d91535e13c526..8f9aab82da64d24645311d1263f9abb006c737eb 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -1,7 +1,6 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading class TDengineCursor(object): @@ -36,7 +35,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection @@ -130,8 +128,8 @@ class TDengineCursor(object): if errno == 0: if CTaosInterface.fieldsCount(self._result) == 0: self._affected_rows += CTaosInterface.affectedRows( - self._result ) - return CTaosInterface.affectedRows(self._result ) + self._result) + return CTaosInterface.affectedRows(self._result) else: self._fields = CTaosInterface.useResult( self._result) @@ -150,6 +148,7 @@ class TDengineCursor(object): """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. """ pass + def fetchmany(self): pass @@ -160,11 +159,26 @@ class TDengineCursor(object): if (dataType.upper() == "TINYINT"): if (self._description[col][1] == FieldType.C_TINYINT): return True + if (dataType.upper() == "TINYINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + return True + if (dataType.upper() == "SMALLINT"): + if (self._description[col][1] == FieldType.C_SMALLINT): + return True + if (dataType.upper() == "SMALLINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + return True if (dataType.upper() == "INT"): if (self._description[col][1] == FieldType.C_INT): return True + if (dataType.upper() == "INT UNSIGNED"): + if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + return True if (dataType.upper() == "BIGINT"): - if (self._description[col][1] == FieldType.C_INT): + if (self._description[col][1] == FieldType.C_BIGINT): + return True + if (dataType.upper() == "BIGINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): return True if (dataType.upper() == "FLOAT"): if (self._description[col][1] == FieldType.C_FLOAT): @@ -184,7 +198,7 @@ class TDengineCursor(object): return False - def fetchall(self): + def fetchall_row(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ if self._result is None or self._fields is None: @@ -193,33 +207,41 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def fetchall_block(self): + + def fetchall(self): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def nextset(self): """ """ diff --git a/src/connector/python/linux/python2/taos/dbapi.py b/src/connector/python/linux/python2/taos/dbapi.py index f1c22bdb512224ac712b78b15ec00207587e65c5..594681ada953abf388e503c23199043cf686e1a3 100644 --- a/src/connector/python/linux/python2/taos/dbapi.py +++ b/src/connector/python/linux/python2/taos/dbapi.py @@ -4,6 +4,7 @@ import time import datetime + class DBAPITypeObject(object): def __init__(self, *values): self.values = values @@ -16,23 +17,28 @@ class DBAPITypeObject(object): else: return -1 + Date = datetime.date Time = datetime.time Timestamp = datetime.datetime + def DataFromTicks(ticks): return Date(*time.localtime(ticks)[:3]) + def TimeFromTicks(ticks): return Time(*time.localtime(ticks)[3:6]) + def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) + Binary = bytes # STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) # BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) # NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) # DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() \ No newline at end of file +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/linux/python2/taos/error.py b/src/connector/python/linux/python2/taos/error.py index 24508a72ed78bb6231187bb6de34d57182e31b22..c584badce8320cd35dc81e8f6b613c56163b1a29 100644 --- a/src/connector/python/linux/python2/taos/error.py +++ b/src/connector/python/linux/python2/taos/error.py @@ -1,35 +1,41 @@ """Python exceptions """ + class Error(Exception): def __init__(self, msg=None, errno=None): self.msg = msg self._full_msg = self.msg self.errno = errno - + def __str__(self): return self._full_msg + class Warning(Exception): """Exception raised for important warnings like data truncations while inserting. """ pass + class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. + """Exception raised for errors that are related to the database interface rather than the database itself. """ pass + class DatabaseError(Error): - """Exception raised for errors that are related to the database. + """Exception raised for errors that are related to the database. """ pass + class DataError(DatabaseError): """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. """ pass + class OperationalError(DatabaseError): """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer """ @@ -41,17 +47,20 @@ class IntegrityError(DatabaseError): """ pass + class InternalError(DatabaseError): """Exception raised when the database encounters an internal error. """ pass + class ProgrammingError(DatabaseError): """Exception raised for programming errors. """ pass + class NotSupportedError(DatabaseError): """Exception raised in case a method or database API was used which is not supported by the database,. """ - pass \ No newline at end of file + pass diff --git a/src/connector/python/linux/python2/taos/subscription.py b/src/connector/python/linux/python2/taos/subscription.py index 2d01395532820c3bd0e068ef7eb3d425eaaa6d78..270d9de09217fc58a389981a3542698dd1c0428a 100644 --- a/src/connector/python/linux/python2/taos/subscription.py +++ b/src/connector/python/linux/python2/taos/subscription.py @@ -1,52 +1,57 @@ from .cinterface import CTaosInterface from .error import * + class TDengineSubscription(object): """TDengine subscription object """ + def __init__(self, sub): self._sub = sub - def consume(self): """Consume rows of a subscription """ if self._sub is None: raise OperationalError("Invalid use of consume") - + result, fields = CTaosInterface.consume(self._sub) buffer = [[] for i in range(len(fields))] while True: block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: break + if num_of_fields == 0: + break for i in range(len(fields)): buffer[i].extend(block[i]) self.fields = fields return list(map(tuple, zip(*buffer))) - - def close(self, keepProgress = True): + def close(self, keepProgress=True): """Close the Subscription. """ if self._sub is None: return False - + CTaosInterface.unsubscribe(self._sub, keepProgress) return True if __name__ == '__main__': from .connection import TDengineConnection - conn = TDengineConnection(host="127.0.0.1", user="root", password="taosdata", database="test") + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) - for i in range(0,10): + for i in range(0, 10): data = sub.consume() for d in data: print(d) sub.close() - conn.close() \ No newline at end of file + conn.close() diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py index 655a12ad135e4eae575061e6069724a12a2a6996..296e79b973fde7c86493565940d63f64e7c6fca3 100644 --- a/src/connector/python/linux/python3/setup.py +++ b/src/connector/python/linux/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.3", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py index 8cf095ea68dda55a071403734d39e9198a71d8a1..973263573808232e4e71dc0158585624a8e7d2ab 100644 --- a/src/connector/python/linux/python3/taos/__init__.py +++ b/src/connector/python/linux/python3/taos/__init__.py @@ -3,12 +3,12 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' __all__ = ['connection', 'cursor'] + def connect(*args, **kwargs): """ Function to return a TDengine connector object diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index fdebe349fe9597260f231adb52022a2ecc9e1063..4367947341edad7b0a9bdbcaec69a7de9801e267 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -4,11 +4,14 @@ from .error import * import math import datetime + def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli/1000.0) + return datetime.datetime.fromtimestamp(milli / 1000.0) + def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro/1000000.0) + return datetime.datetime.fromtimestamp(micro / 1000000.0) + def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row @@ -18,168 +21,309 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) + def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] - + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] + def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append( tmpstr.value.decode() ) + res.append(tmpstr.value.decode()) else: - res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) - return res + return res + def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows > 0: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) return res + def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows >= 0: for i in range(abs(num_of_rows)): try: - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode() ) + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res + _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } _CONVERT_FUNC_BLOCK = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python_block + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } # Corresponding TAOS_FIELD structure in C + + class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), ('type', ctypes.c_char), ('bytes', ctypes.c_short)] # C interface class + + class CTaosInterface(object): libtaos = ctypes.CDLL('libtaos.so') @@ -216,7 +360,7 @@ class CTaosInterface(object): except AttributeError: raise AttributeError("config is expected as a str") - if config != None: + if config is not None: CTaosInterface.libtaos.taos_options(3, self._config) CTaosInterface.libtaos.taos_init() @@ -227,7 +371,13 @@ class CTaosInterface(object): """ return self._config - def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): ''' Function to connect to server @@ -236,7 +386,7 @@ class CTaosInterface(object): # host try: _host = ctypes.c_char_p(host.encode( - "utf-8")) if host != None else ctypes.c_char_p(None) + "utf-8")) if host is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("host is expected as a str") @@ -255,7 +405,7 @@ class CTaosInterface(object): # db try: _db = ctypes.c_char_p( - db.encode("utf-8")) if db != None else ctypes.c_char_p(None) + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("db is expected as a str") @@ -268,11 +418,11 @@ class CTaosInterface(object): connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( _host, _user, _password, _db, _port)) - if connection.value == None: + if connection.value is None: print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - #else: + # else: # print('connect to TDengine success') return connection @@ -293,7 +443,8 @@ class CTaosInterface(object): @rtype: 0 on success and -1 on failure ''' try: - return CTaosInterface.libtaos.taos_query(connection, ctypes.c_char_p(sql.encode('utf-8'))) + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) except AttributeError: raise AttributeError("sql is expected as a string") # finally: @@ -308,7 +459,7 @@ class CTaosInterface(object): @staticmethod def subscribe(connection, restart, topic, sql, interval): """Create a subscription - @restart boolean, + @restart boolean, @sql string, sql statement for data query, must be a 'select' statement. @topic string, name of this subscription """ @@ -360,35 +511,49 @@ class CTaosInterface(object): result, ctypes.byref(pblock)) if num_of_rows == 0: return None, 0 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) + @staticmethod def fetchRow(result, fields): pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock : + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: num_of_rows = 1 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError("Invalid data type returned from database") + raise DatabaseError( + "Invalid data type returned from database") if data is None: blocks[i] = [None] else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) else: return None, 0 return blocks, abs(num_of_rows) @@ -435,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -443,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') diff --git a/src/connector/python/linux/python3/taos/connection.py b/src/connector/python/linux/python3/taos/connection.py index 552250f1164ced467cd29b5084524985aca8848b..f6c395342c9c39a24bda6022f0ed36cb7bfe045b 100644 --- a/src/connector/python/linux/python3/taos/connection.py +++ b/src/connector/python/linux/python3/taos/connection.py @@ -2,9 +2,11 @@ from .cursor import TDengineCursor from .subscription import TDengineSubscription from .cinterface import CTaosInterface + class TDengineConnection(object): """ TDengine connection object """ + def __init__(self, *args, **kwargs): self._conn = None self._host = None @@ -29,7 +31,7 @@ class TDengineConnection(object): # password if 'password' in kwargs: self._password = kwargs['password'] - + # database if 'database' in kwargs: self._database = kwargs['database'] @@ -43,7 +45,12 @@ class TDengineConnection(object): self._config = kwargs['config'] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) def close(self): """Close current connection. @@ -55,7 +62,8 @@ class TDengineConnection(object): """ if self._conn is None: return None - sub = CTaosInterface.subscribe(self._conn, restart, topic, sql, interval) + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) return TDengineSubscription(sub) def cursor(self): @@ -80,7 +88,8 @@ class TDengineConnection(object): """ pass + if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') conn.close() - print("Hello world") \ No newline at end of file + print("Hello world") diff --git a/src/connector/python/linux/python3/taos/constants.py b/src/connector/python/linux/python3/taos/constants.py index feb7050a40b67f88a6d7ca859764fbbc6b36af1c..93466f5184a6bf37c2e1c915a00aa5c5e91d1801 100644 --- a/src/connector/python/linux/python3/taos/constants.py +++ b/src/connector/python/linux/python3/taos/constants.py @@ -3,6 +3,7 @@ from .dbapi import * + class FieldType(object): """TDengine Field Types """ @@ -18,13 +19,21 @@ class FieldType(object): C_BINARY = 8 C_TIMESTAMP = 9 C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index 0ce20f0edaabe5cf0d2bb6efa3e031f51779f03d..32dc0ea3c3676c61c65bf92b4fb04a5373314126 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -1,10 +1,10 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading # querySeqNum = 0 + class TDengineCursor(object): """Database cursor which is used to manage the context of a fetch operation. @@ -37,7 +37,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection @@ -104,12 +103,6 @@ class TDengineCursor(object): def execute(self, operation, params=None): """Prepare and execute a database operation (query or command). """ - # if threading.get_ident() != self._threadId: - # info ="Cursor execute:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None - if not operation: return None @@ -137,8 +130,8 @@ class TDengineCursor(object): if errno == 0: if CTaosInterface.fieldsCount(self._result) == 0: self._affected_rows += CTaosInterface.affectedRows( - self._result ) - return CTaosInterface.affectedRows(self._result ) + self._result) + return CTaosInterface.affectedRows(self._result) else: self._fields = CTaosInterface.useResult( self._result) @@ -168,11 +161,26 @@ class TDengineCursor(object): if (dataType.upper() == "TINYINT"): if (self._description[col][1] == FieldType.C_TINYINT): return True + if (dataType.upper() == "TINYINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + return True + if (dataType.upper() == "SMALLINT"): + if (self._description[col][1] == FieldType.C_SMALLINT): + return True + if (dataType.upper() == "SMALLINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + return True if (dataType.upper() == "INT"): if (self._description[col][1] == FieldType.C_INT): return True + if (dataType.upper() == "INT UNSIGNED"): + if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + return True if (dataType.upper() == "BIGINT"): - if (self._description[col][1] == FieldType.C_INT): + if (self._description[col][1] == FieldType.C_BIGINT): + return True + if (dataType.upper() == "BIGINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): return True if (dataType.upper() == "FLOAT"): if (self._description[col][1] == FieldType.C_FLOAT): @@ -192,7 +200,7 @@ class TDengineCursor(object): return False - def fetchall(self): + def fetchall_row(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ if self._result is None or self._fields is None: @@ -201,10 +209,13 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -212,22 +223,27 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def fetchall_block(self): + def fetchall(self): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) - if num_of_fields == 0: break + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def nextset(self): """ """ @@ -256,16 +272,9 @@ class TDengineCursor(object): def _handle_result(self): """Handle the return result from query. """ - # if threading.get_ident() != self._threadId: - # info = "Cursor handleresult:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None - self._description = [] for ele in self._fields: self._description.append( (ele['name'], ele['type'], None, None, None, None, False)) return self._result - diff --git a/src/connector/python/linux/python3/taos/dbapi.py b/src/connector/python/linux/python3/taos/dbapi.py index f1c22bdb512224ac712b78b15ec00207587e65c5..594681ada953abf388e503c23199043cf686e1a3 100644 --- a/src/connector/python/linux/python3/taos/dbapi.py +++ b/src/connector/python/linux/python3/taos/dbapi.py @@ -4,6 +4,7 @@ import time import datetime + class DBAPITypeObject(object): def __init__(self, *values): self.values = values @@ -16,23 +17,28 @@ class DBAPITypeObject(object): else: return -1 + Date = datetime.date Time = datetime.time Timestamp = datetime.datetime + def DataFromTicks(ticks): return Date(*time.localtime(ticks)[:3]) + def TimeFromTicks(ticks): return Time(*time.localtime(ticks)[3:6]) + def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) + Binary = bytes # STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) # BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) # NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) # DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() \ No newline at end of file +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/linux/python3/taos/error.py b/src/connector/python/linux/python3/taos/error.py index 24508a72ed78bb6231187bb6de34d57182e31b22..c584badce8320cd35dc81e8f6b613c56163b1a29 100644 --- a/src/connector/python/linux/python3/taos/error.py +++ b/src/connector/python/linux/python3/taos/error.py @@ -1,35 +1,41 @@ """Python exceptions """ + class Error(Exception): def __init__(self, msg=None, errno=None): self.msg = msg self._full_msg = self.msg self.errno = errno - + def __str__(self): return self._full_msg + class Warning(Exception): """Exception raised for important warnings like data truncations while inserting. """ pass + class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. + """Exception raised for errors that are related to the database interface rather than the database itself. """ pass + class DatabaseError(Error): - """Exception raised for errors that are related to the database. + """Exception raised for errors that are related to the database. """ pass + class DataError(DatabaseError): """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. """ pass + class OperationalError(DatabaseError): """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer """ @@ -41,17 +47,20 @@ class IntegrityError(DatabaseError): """ pass + class InternalError(DatabaseError): """Exception raised when the database encounters an internal error. """ pass + class ProgrammingError(DatabaseError): """Exception raised for programming errors. """ pass + class NotSupportedError(DatabaseError): """Exception raised in case a method or database API was used which is not supported by the database,. """ - pass \ No newline at end of file + pass diff --git a/src/connector/python/linux/python3/taos/subscription.py b/src/connector/python/linux/python3/taos/subscription.py index d3cf10d5ada578687689b94454378dd543368e3e..270d9de09217fc58a389981a3542698dd1c0428a 100644 --- a/src/connector/python/linux/python3/taos/subscription.py +++ b/src/connector/python/linux/python3/taos/subscription.py @@ -1,32 +1,33 @@ from .cinterface import CTaosInterface from .error import * + class TDengineSubscription(object): """TDengine subscription object """ + def __init__(self, sub): self._sub = sub - def consume(self): """Consume rows of a subscription """ if self._sub is None: raise OperationalError("Invalid use of consume") - + result, fields = CTaosInterface.consume(self._sub) buffer = [[] for i in range(len(fields))] while True: block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: break + if num_of_fields == 0: + break for i in range(len(fields)): buffer[i].extend(block[i]) self.fields = fields return list(map(tuple, zip(*buffer))) - - def close(self, keepProgress = True): + def close(self, keepProgress=True): """Close the Subscription. """ if self._sub is None: @@ -38,15 +39,19 @@ class TDengineSubscription(object): if __name__ == '__main__': from .connection import TDengineConnection - conn = TDengineConnection(host="127.0.0.1", user="root", password="taosdata", database="test") + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) - for i in range(0,10): + for i in range(0, 10): data = sub.consume() for d in data: print(d) sub.close() - conn.close() \ No newline at end of file + conn.close() diff --git a/src/connector/python/osx/python3/LICENSE b/src/connector/python/osx/python3/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..79a9d730868bfe5d3fa01d679a4abfe9ee7811f0 --- /dev/null +++ b/src/connector/python/osx/python3/LICENSE @@ -0,0 +1,12 @@ + Copyright (c) 2019 TAOS Data, Inc. + +This program is free software: you can use, redistribute, and/or modify +it under the terms of the GNU Affero General Public License, version 3 +or later ("AGPL"), as published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . diff --git a/src/connector/python/osx/python3/README.md b/src/connector/python/osx/python3/README.md new file mode 100644 index 0000000000000000000000000000000000000000..70db6bba13a8b52b9f707400b80d1302542dbc34 --- /dev/null +++ b/src/connector/python/osx/python3/README.md @@ -0,0 +1 @@ +# TDengine python client interface \ No newline at end of file diff --git a/src/connector/python/osx/python3/setup.py b/src/connector/python/osx/python3/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..9bce1a976febcec457c2b3f8ade0bc6d546307c4 --- /dev/null +++ b/src/connector/python/osx/python3/setup.py @@ -0,0 +1,20 @@ +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +setuptools.setup( + name="taos", + version="2.0.7", + author="Taosdata Inc.", + author_email="support@taosdata.com", + description="TDengine python client package", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/pypa/sampleproject", + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: MacOS X", + ], +) diff --git a/src/connector/python/osx/python3/taos/__init__.py b/src/connector/python/osx/python3/taos/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..973263573808232e4e71dc0158585624a8e7d2ab --- /dev/null +++ b/src/connector/python/osx/python3/taos/__init__.py @@ -0,0 +1,24 @@ + +from .connection import TDengineConnection +from .cursor import TDengineCursor + +# Globals +threadsafety = 0 +paramstyle = 'pyformat' + +__all__ = ['connection', 'cursor'] + + +def connect(*args, **kwargs): + """ Function to return a TDengine connector object + + Current supporting keyword parameters: + @dsn: Data source name as string + @user: Username as string(optional) + @password: Password as string(optional) + @host: Hostname(optional) + @database: Database name(optional) + + @rtype: TDengineConnector + """ + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/osx/python3/taos/cinterface.py b/src/connector/python/osx/python3/taos/cinterface.py new file mode 100644 index 0000000000000000000000000000000000000000..dca9bd42e8733616f34654542ba8c2c3ea3ece9d --- /dev/null +++ b/src/connector/python/osx/python3/taos/cinterface.py @@ -0,0 +1,642 @@ +import ctypes +from .constants import FieldType +from .error import * +import math +import datetime + + +def _convert_millisecond_to_datetime(milli): + return datetime.datetime.fromtimestamp(milli / 1000.0) + + +def _convert_microsecond_to_datetime(micro): + return datetime.datetime.fromtimestamp(micro / 1000000.0) + + +def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C bool row to python row + """ + _timestamp_converter = _convert_millisecond_to_datetime + if micro: + _timestamp_converter = _convert_microsecond_to_datetime + + if num_of_rows > 0: + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) + else: + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) + + +def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C bool row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + + +def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + +def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + + +def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + + +def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] + + +def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C float row to python row + """ + if num_of_rows > 0: + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + else: + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + + +def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C double row to python row + """ + if num_of_rows > 0: + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + else: + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + + +def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + assert(nbytes is not None) + if num_of_rows > 0: + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + else: + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + + +def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res = [] + for i in range(abs(num_of_rows)): + try: + if num_of_rows >= 0: + tmpstr = ctypes.c_char_p(data) + res.append(tmpstr.value.decode()) + else: + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) + except ValueError: + res.append(None) + + return res + + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + assert(nbytes is not None) + res = [] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) + except ValueError: + res.append(None) + return res + + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res = [] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) + except ValueError: + res.append(None) + return res + + +_CONVERT_FUNC = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python +} + +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python +} + +# Corresponding TAOS_FIELD structure in C + + +class TaosField(ctypes.Structure): + _fields_ = [('name', ctypes.c_char * 65), + ('type', ctypes.c_char), + ('bytes', ctypes.c_short)] + +# C interface class + + +class CTaosInterface(object): + + libtaos = ctypes.CDLL('libtaos.dylib') + + libtaos.taos_fetch_fields.restype = ctypes.POINTER(TaosField) + libtaos.taos_init.restype = None + libtaos.taos_connect.restype = ctypes.c_void_p + #libtaos.taos_use_result.restype = ctypes.c_void_p + libtaos.taos_fetch_row.restype = ctypes.POINTER(ctypes.c_void_p) + libtaos.taos_errstr.restype = ctypes.c_char_p + libtaos.taos_subscribe.restype = ctypes.c_void_p + libtaos.taos_consume.restype = ctypes.c_void_p + libtaos.taos_fetch_lengths.restype = ctypes.c_void_p + libtaos.taos_free_result.restype = None + libtaos.taos_errno.restype = ctypes.c_int + libtaos.taos_query.restype = ctypes.POINTER(ctypes.c_void_p) + + def __init__(self, config=None): + ''' + Function to initialize the class + @host : str, hostname to connect + @user : str, username to connect to server + @password : str, password to connect to server + @db : str, default db to use when log in + @config : str, config directory + + @rtype : None + ''' + if config is None: + self._config = ctypes.c_char_p(None) + else: + try: + self._config = ctypes.c_char_p(config.encode('utf-8')) + except AttributeError: + raise AttributeError("config is expected as a str") + + if config is not None: + CTaosInterface.libtaos.taos_options(3, self._config) + + CTaosInterface.libtaos.taos_init() + + @property + def config(self): + """ Get current config + """ + return self._config + + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): + ''' + Function to connect to server + + @rtype: c_void_p, TDengine handle + ''' + # host + try: + _host = ctypes.c_char_p(host.encode( + "utf-8")) if host is not None else ctypes.c_char_p(None) + except AttributeError: + raise AttributeError("host is expected as a str") + + # user + try: + _user = ctypes.c_char_p(user.encode("utf-8")) + except AttributeError: + raise AttributeError("user is expected as a str") + + # password + try: + _password = ctypes.c_char_p(password.encode("utf-8")) + except AttributeError: + raise AttributeError("password is expected as a str") + + # db + try: + _db = ctypes.c_char_p( + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) + except AttributeError: + raise AttributeError("db is expected as a str") + + # port + try: + _port = ctypes.c_int(port) + except TypeError: + raise TypeError("port is expected as an int") + + connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( + _host, _user, _password, _db, _port)) + + if connection.value is None: + print('connect to TDengine failed') + raise ConnectionError("connect to TDengine failed") + # sys.exit(1) + # else: + # print('connect to TDengine success') + + return connection + + @staticmethod + def close(connection): + '''Close the TDengine handle + ''' + CTaosInterface.libtaos.taos_close(connection) + #print('connection is closed') + + @staticmethod + def query(connection, sql): + '''Run SQL + + @sql: str, sql string to run + + @rtype: 0 on success and -1 on failure + ''' + try: + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) + except AttributeError: + raise AttributeError("sql is expected as a string") + # finally: + # CTaosInterface.libtaos.close(connection) + + @staticmethod + def affectedRows(result): + """The affected rows after runing query + """ + return CTaosInterface.libtaos.taos_affected_rows(result) + + @staticmethod + def subscribe(connection, restart, topic, sql, interval): + """Create a subscription + @restart boolean, + @sql string, sql statement for data query, must be a 'select' statement. + @topic string, name of this subscription + """ + return ctypes.c_void_p(CTaosInterface.libtaos.taos_subscribe( + connection, + 1 if restart else 0, + ctypes.c_char_p(topic.encode('utf-8')), + ctypes.c_char_p(sql.encode('utf-8')), + None, + None, + interval)) + + @staticmethod + def consume(sub): + """Consume data of a subscription + """ + result = ctypes.c_void_p(CTaosInterface.libtaos.taos_consume(sub)) + fields = [] + pfields = CTaosInterface.fetchFields(result) + for i in range(CTaosInterface.libtaos.taos_num_fields(result)): + fields.append({'name': pfields[i].name.decode('utf-8'), + 'bytes': pfields[i].bytes, + 'type': ord(pfields[i].type)}) + return result, fields + + @staticmethod + def unsubscribe(sub, keepProgress): + """Cancel a subscription + """ + CTaosInterface.libtaos.taos_unsubscribe(sub, 1 if keepProgress else 0) + + @staticmethod + def useResult(result): + '''Use result after calling self.query + ''' + fields = [] + pfields = CTaosInterface.fetchFields(result) + for i in range(CTaosInterface.fieldsCount(result)): + fields.append({'name': pfields[i].name.decode('utf-8'), + 'bytes': pfields[i].bytes, + 'type': ord(pfields[i].type)}) + + return fields + + @staticmethod + def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + + @staticmethod + def fetchRow(result, fields): + pblock = ctypes.c_void_p(0) + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: + num_of_rows = 1 + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC: + raise DatabaseError( + "Invalid data type returned from database") + if data is None: + blocks[i] = [None] + else: + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) + else: + return None, 0 + return blocks, abs(num_of_rows) + + @staticmethod + def freeResult(result): + CTaosInterface.libtaos.taos_free_result(result) + result.value = None + + @staticmethod + def fieldsCount(result): + return CTaosInterface.libtaos.taos_field_count(result) + + @staticmethod + def fetchFields(result): + return CTaosInterface.libtaos.taos_fetch_fields(result) + + # @staticmethod + # def fetchRow(result, fields): + # l = [] + # row = CTaosInterface.libtaos.taos_fetch_row(result) + # if not row: + # return None + + # for i in range(len(fields)): + # l.append(CTaosInterface.getDataValue( + # row[i], fields[i]['type'], fields[i]['bytes'])) + + # return tuple(l) + + # @staticmethod + # def getDataValue(data, dtype, byte): + # ''' + # ''' + # if not data: + # return None + + # if (dtype == CTaosInterface.TSDB_DATA_TYPE_BOOL): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TINYINT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_SMALLINT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): + # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] + # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): + # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') + + @staticmethod + def errno(result): + """Return the error number. + """ + return CTaosInterface.libtaos.taos_errno(result) + + @staticmethod + def errStr(result): + """Return the error styring + """ + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') + + +if __name__ == '__main__': + cinter = CTaosInterface() + conn = cinter.connect() + result = cinter.query(conn, 'show databases') + + print('Query Affected rows: {}'.format(cinter.affectedRows(result))) + + fields = CTaosInterface.useResult(result) + + data, num_of_rows = CTaosInterface.fetchBlock(result, fields) + + print(data) + + cinter.freeResult(result) + cinter.close(conn) diff --git a/src/connector/python/osx/python3/taos/connection.py b/src/connector/python/osx/python3/taos/connection.py new file mode 100644 index 0000000000000000000000000000000000000000..f6c395342c9c39a24bda6022f0ed36cb7bfe045b --- /dev/null +++ b/src/connector/python/osx/python3/taos/connection.py @@ -0,0 +1,95 @@ +from .cursor import TDengineCursor +from .subscription import TDengineSubscription +from .cinterface import CTaosInterface + + +class TDengineConnection(object): + """ TDengine connection object + """ + + def __init__(self, *args, **kwargs): + self._conn = None + self._host = None + self._user = "root" + self._password = "taosdata" + self._database = None + self._port = 0 + self._config = None + self._chandle = None + + self.config(**kwargs) + + def config(self, **kwargs): + # host + if 'host' in kwargs: + self._host = kwargs['host'] + + # user + if 'user' in kwargs: + self._user = kwargs['user'] + + # password + if 'password' in kwargs: + self._password = kwargs['password'] + + # database + if 'database' in kwargs: + self._database = kwargs['database'] + + # port + if 'port' in kwargs: + self._port = kwargs['port'] + + # config + if 'config' in kwargs: + self._config = kwargs['config'] + + self._chandle = CTaosInterface(self._config) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) + + def close(self): + """Close current connection. + """ + return CTaosInterface.close(self._conn) + + def subscribe(self, restart, topic, sql, interval): + """Create a subscription. + """ + if self._conn is None: + return None + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) + return TDengineSubscription(sub) + + def cursor(self): + """Return a new Cursor object using the connection. + """ + return TDengineCursor(self) + + def commit(self): + """Commit any pending transaction to the database. + + Since TDengine do not support transactions, the implement is void functionality. + """ + pass + + def rollback(self): + """Void functionality + """ + pass + + def clear_result_set(self): + """Clear unused result set on this connection. + """ + pass + + +if __name__ == "__main__": + conn = TDengineConnection(host='192.168.1.107') + conn.close() + print("Hello world") diff --git a/src/connector/python/osx/python3/taos/constants.py b/src/connector/python/osx/python3/taos/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..93466f5184a6bf37c2e1c915a00aa5c5e91d1801 --- /dev/null +++ b/src/connector/python/osx/python3/taos/constants.py @@ -0,0 +1,42 @@ +"""Constants in TDengine python +""" + +from .dbapi import * + + +class FieldType(object): + """TDengine Field Types + """ + # type_code + C_NULL = 0 + C_BOOL = 1 + C_TINYINT = 2 + C_SMALLINT = 3 + C_INT = 4 + C_BIGINT = 5 + C_FLOAT = 6 + C_DOUBLE = 7 + C_BINARY = 8 + C_TIMESTAMP = 9 + C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 + # NULL value definition + # NOTE: These values should change according to C definition in tsdb.h + C_BOOL_NULL = 0x02 + C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 + C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 + C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 + C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 + C_FLOAT_NULL = float('nan') + C_DOUBLE_NULL = float('nan') + C_BINARY_NULL = bytearray([int('0xff', 16)]) + # Timestamp precision definition + C_TIMESTAMP_MILLI = 0 + C_TIMESTAMP_MICRO = 1 diff --git a/src/connector/python/osx/python3/taos/cursor.py b/src/connector/python/osx/python3/taos/cursor.py new file mode 100644 index 0000000000000000000000000000000000000000..32dc0ea3c3676c61c65bf92b4fb04a5373314126 --- /dev/null +++ b/src/connector/python/osx/python3/taos/cursor.py @@ -0,0 +1,280 @@ +from .cinterface import CTaosInterface +from .error import * +from .constants import FieldType + +# querySeqNum = 0 + + +class TDengineCursor(object): + """Database cursor which is used to manage the context of a fetch operation. + + Attributes: + .description: Read-only attribute consists of 7-item sequences: + + > name (mondatory) + > type_code (mondatory) + > display_size + > internal_size + > precision + > scale + > null_ok + + This attribute will be None for operations that do not return rows or + if the cursor has not had an operation invoked via the .execute*() method yet. + + .rowcount:This read-only attribute specifies the number of rows that the last + .execute*() produced (for DQL statements like SELECT) or affected + """ + + def __init__(self, connection=None): + self._description = [] + self._rowcount = -1 + self._connection = None + self._result = None + self._fields = None + self._block = None + self._block_rows = -1 + self._block_iter = 0 + self._affected_rows = 0 + self._logfile = "" + + if connection is not None: + self._connection = connection + + def __iter__(self): + return self + + def __next__(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetch iterator") + + if self._block_rows <= self._block_iter: + block, self._block_rows = CTaosInterface.fetchRow( + self._result, self._fields) + if self._block_rows == 0: + raise StopIteration + self._block = list(map(tuple, zip(*block))) + self._block_iter = 0 + + data = self._block[self._block_iter] + self._block_iter += 1 + + return data + + @property + def description(self): + """Return the description of the object. + """ + return self._description + + @property + def rowcount(self): + """Return the rowcount of the object + """ + return self._rowcount + + @property + def affected_rows(self): + """Return the rowcount of insertion + """ + return self._affected_rows + + def callproc(self, procname, *args): + """Call a stored database procedure with the given name. + + Void functionality since no stored procedures. + """ + pass + + def log(self, logfile): + self._logfile = logfile + + def close(self): + """Close the cursor. + """ + if self._connection is None: + return False + + self._reset_result() + self._connection = None + + return True + + def execute(self, operation, params=None): + """Prepare and execute a database operation (query or command). + """ + if not operation: + return None + + if not self._connection: + # TODO : change the exception raised here + raise ProgrammingError("Cursor is not connected") + + self._reset_result() + + stmt = operation + if params is not None: + pass + + # global querySeqNum + # querySeqNum += 1 + # localSeqNum = querySeqNum # avoid raice condition + # print(" >> Exec Query ({}): {}".format(localSeqNum, str(stmt))) + self._result = CTaosInterface.query(self._connection._conn, stmt) + # print(" << Query ({}) Exec Done".format(localSeqNum)) + if (self._logfile): + with open(self._logfile, "a") as logfile: + logfile.write("%s;\n" % operation) + + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno == 0: + if CTaosInterface.fieldsCount(self._result) == 0: + self._affected_rows += CTaosInterface.affectedRows( + self._result) + return CTaosInterface.affectedRows(self._result) + else: + self._fields = CTaosInterface.useResult( + self._result) + return self._handle_result() + else: + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + + def executemany(self, operation, seq_of_parameters): + """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. + """ + pass + + def fetchone(self): + """Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. + """ + pass + + def fetchmany(self): + pass + + def istype(self, col, dataType): + if (dataType.upper() == "BOOL"): + if (self._description[col][1] == FieldType.C_BOOL): + return True + if (dataType.upper() == "TINYINT"): + if (self._description[col][1] == FieldType.C_TINYINT): + return True + if (dataType.upper() == "TINYINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_TINYINT_UNSIGNED): + return True + if (dataType.upper() == "SMALLINT"): + if (self._description[col][1] == FieldType.C_SMALLINT): + return True + if (dataType.upper() == "SMALLINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_SMALLINT_UNSIGNED): + return True + if (dataType.upper() == "INT"): + if (self._description[col][1] == FieldType.C_INT): + return True + if (dataType.upper() == "INT UNSIGNED"): + if (self._description[col][1] == FieldType.C_INT_UNSIGNED): + return True + if (dataType.upper() == "BIGINT"): + if (self._description[col][1] == FieldType.C_BIGINT): + return True + if (dataType.upper() == "BIGINT UNSIGNED"): + if (self._description[col][1] == FieldType.C_BIGINT_UNSIGNED): + return True + if (dataType.upper() == "FLOAT"): + if (self._description[col][1] == FieldType.C_FLOAT): + return True + if (dataType.upper() == "DOUBLE"): + if (self._description[col][1] == FieldType.C_DOUBLE): + return True + if (dataType.upper() == "BINARY"): + if (self._description[col][1] == FieldType.C_BINARY): + return True + if (dataType.upper() == "TIMESTAMP"): + if (self._description[col][1] == FieldType.C_TIMESTAMP): + return True + if (dataType.upper() == "NCHAR"): + if (self._description[col][1] == FieldType.C_NCHAR): + return True + + return False + + def fetchall_row(self): + """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. + """ + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def fetchall(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def nextset(self): + """ + """ + pass + + def setinputsize(self, sizes): + pass + + def setutputsize(self, size, column=None): + pass + + def _reset_result(self): + """Reset the result to unused version. + """ + self._description = [] + self._rowcount = -1 + if self._result is not None: + CTaosInterface.freeResult(self._result) + self._result = None + self._fields = None + self._block = None + self._block_rows = -1 + self._block_iter = 0 + self._affected_rows = 0 + + def _handle_result(self): + """Handle the return result from query. + """ + self._description = [] + for ele in self._fields: + self._description.append( + (ele['name'], ele['type'], None, None, None, None, False)) + + return self._result diff --git a/src/connector/python/osx/python3/taos/dbapi.py b/src/connector/python/osx/python3/taos/dbapi.py new file mode 100644 index 0000000000000000000000000000000000000000..594681ada953abf388e503c23199043cf686e1a3 --- /dev/null +++ b/src/connector/python/osx/python3/taos/dbapi.py @@ -0,0 +1,44 @@ +"""Type Objects and Constructors. +""" + +import time +import datetime + + +class DBAPITypeObject(object): + def __init__(self, *values): + self.values = values + + def __com__(self, other): + if other in self.values: + return 0 + if other < self.values: + return 1 + else: + return -1 + + +Date = datetime.date +Time = datetime.time +Timestamp = datetime.datetime + + +def DataFromTicks(ticks): + return Date(*time.localtime(ticks)[:3]) + + +def TimeFromTicks(ticks): + return Time(*time.localtime(ticks)[3:6]) + + +def TimestampFromTicks(ticks): + return Timestamp(*time.localtime(ticks)[:6]) + + +Binary = bytes + +# STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) +# BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) +# NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) +# DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/osx/python3/taos/error.py b/src/connector/python/osx/python3/taos/error.py new file mode 100644 index 0000000000000000000000000000000000000000..c584badce8320cd35dc81e8f6b613c56163b1a29 --- /dev/null +++ b/src/connector/python/osx/python3/taos/error.py @@ -0,0 +1,66 @@ +"""Python exceptions +""" + + +class Error(Exception): + def __init__(self, msg=None, errno=None): + self.msg = msg + self._full_msg = self.msg + self.errno = errno + + def __str__(self): + return self._full_msg + + +class Warning(Exception): + """Exception raised for important warnings like data truncations while inserting. + """ + pass + + +class InterfaceError(Error): + """Exception raised for errors that are related to the database interface rather than the database itself. + """ + pass + + +class DatabaseError(Error): + """Exception raised for errors that are related to the database. + """ + pass + + +class DataError(DatabaseError): + """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. + """ + pass + + +class OperationalError(DatabaseError): + """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer + """ + pass + + +class IntegrityError(DatabaseError): + """Exception raised when the relational integrity of the database is affected. + """ + pass + + +class InternalError(DatabaseError): + """Exception raised when the database encounters an internal error. + """ + pass + + +class ProgrammingError(DatabaseError): + """Exception raised for programming errors. + """ + pass + + +class NotSupportedError(DatabaseError): + """Exception raised in case a method or database API was used which is not supported by the database,. + """ + pass diff --git a/src/connector/python/osx/python3/taos/subscription.py b/src/connector/python/osx/python3/taos/subscription.py new file mode 100644 index 0000000000000000000000000000000000000000..270d9de09217fc58a389981a3542698dd1c0428a --- /dev/null +++ b/src/connector/python/osx/python3/taos/subscription.py @@ -0,0 +1,57 @@ +from .cinterface import CTaosInterface +from .error import * + + +class TDengineSubscription(object): + """TDengine subscription object + """ + + def __init__(self, sub): + self._sub = sub + + def consume(self): + """Consume rows of a subscription + """ + if self._sub is None: + raise OperationalError("Invalid use of consume") + + result, fields = CTaosInterface.consume(self._sub) + buffer = [[] for i in range(len(fields))] + while True: + block, num_of_fields = CTaosInterface.fetchBlock(result, fields) + if num_of_fields == 0: + break + for i in range(len(fields)): + buffer[i].extend(block[i]) + + self.fields = fields + return list(map(tuple, zip(*buffer))) + + def close(self, keepProgress=True): + """Close the Subscription. + """ + if self._sub is None: + return False + + CTaosInterface.unsubscribe(self._sub, keepProgress) + return True + + +if __name__ == '__main__': + from .connection import TDengineConnection + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") + + # Generate a cursor object to run SQL commands + sub = conn.subscribe(True, "test", "select * from meters;", 1000) + + for i in range(0, 10): + data = sub.consume() + for d in data: + print(d) + + sub.close() + conn.close() diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py index 5ddbe83011daa9d408e22dbeda7172466329921e..47d374fe67673172596824f68ed495f68941bb51 100644 --- a/src/connector/python/windows/python2/setup.py +++ b/src/connector/python/windows/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.3", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python2/taos/__init__.py b/src/connector/python/windows/python2/taos/__init__.py index 62e0536b6fca63a0aca0b6be3673104c27c631c6..973263573808232e4e71dc0158585624a8e7d2ab 100644 --- a/src/connector/python/windows/python2/taos/__init__.py +++ b/src/connector/python/windows/python2/taos/__init__.py @@ -3,12 +3,12 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' __all__ = ['connection', 'cursor'] + def connect(*args, **kwargs): """ Function to return a TDengine connector object @@ -21,4 +21,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) \ No newline at end of file + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index 14f4f49be8ae719da16dbed7e79631db7ef9689e..ec72474df93a4d13aa13256611044c5bc140f2d3 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -4,11 +4,14 @@ from .error import * import math import datetime + def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli/1000.0) + return datetime.datetime.fromtimestamp(milli / 1000.0) + def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro/1000000.0) + return datetime.datetime.fromtimestamp(micro / 1000000.0) + def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row @@ -18,168 +21,309 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) + def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] - + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] + def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append( tmpstr.value.decode() ) + res.append(tmpstr.value.decode()) else: - res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) - return res + return res + def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows > 0: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) return res + def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows >= 0: for i in range(abs(num_of_rows)): try: - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode() ) + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res + _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } _CONVERT_FUNC_BLOCK = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python_block + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } # Corresponding TAOS_FIELD structure in C + + class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), ('type', ctypes.c_char), ('bytes', ctypes.c_short)] # C interface class + + class CTaosInterface(object): libtaos = ctypes.windll.LoadLibrary('taos') @@ -216,7 +360,7 @@ class CTaosInterface(object): except AttributeError: raise AttributeError("config is expected as a str") - if config != None: + if config is not None: CTaosInterface.libtaos.taos_options(3, self._config) CTaosInterface.libtaos.taos_init() @@ -227,7 +371,13 @@ class CTaosInterface(object): """ return self._config - def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): ''' Function to connect to server @@ -236,7 +386,7 @@ class CTaosInterface(object): # host try: _host = ctypes.c_char_p(host.encode( - "utf-8")) if host != None else ctypes.c_char_p(None) + "utf-8")) if host is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("host is expected as a str") @@ -255,7 +405,7 @@ class CTaosInterface(object): # db try: _db = ctypes.c_char_p( - db.encode("utf-8")) if db != None else ctypes.c_char_p(None) + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("db is expected as a str") @@ -268,11 +418,11 @@ class CTaosInterface(object): connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( _host, _user, _password, _db, _port)) - if connection.value == None: + if connection.value is None: print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - #else: + # else: # print('connect to TDengine success') return connection @@ -293,12 +443,13 @@ class CTaosInterface(object): @rtype: 0 on success and -1 on failure ''' try: - return CTaosInterface.libtaos.taos_query(connection, ctypes.c_char_p(sql.encode('utf-8'))) + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) except AttributeError: raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -308,7 +459,7 @@ class CTaosInterface(object): @staticmethod def subscribe(connection, restart, topic, sql, interval): """Create a subscription - @restart boolean, + @restart boolean, @sql string, sql statement for data query, must be a 'select' statement. @topic string, name of this subscription """ @@ -360,35 +511,49 @@ class CTaosInterface(object): result, ctypes.byref(pblock)) if num_of_rows == 0: return None, 0 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) + @staticmethod def fetchRow(result, fields): pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock : + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: num_of_rows = 1 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError("Invalid data type returned from database") + raise DatabaseError( + "Invalid data type returned from database") if data is None: blocks[i] = [None] else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) else: return None, 0 return blocks, abs(num_of_rows) @@ -435,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -443,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') diff --git a/src/connector/python/windows/python2/taos/connection.py b/src/connector/python/windows/python2/taos/connection.py index d9576a553b810a975429b2cefc03e5e60f240a88..5729d01c6df8c0e58086726c4001467811e9fee5 100644 --- a/src/connector/python/windows/python2/taos/connection.py +++ b/src/connector/python/windows/python2/taos/connection.py @@ -2,9 +2,11 @@ from .cursor import TDengineCursor from .subscription import TDengineSubscription from .cinterface import CTaosInterface + class TDengineConnection(object): """ TDengine connection object """ + def __init__(self, *args, **kwargs): self._conn = None self._host = None @@ -30,7 +32,7 @@ class TDengineConnection(object): # password if 'password' in kwargs: self._password = kwargs['password'] - + # database if 'database' in kwargs: self._database = kwargs['database'] @@ -44,7 +46,12 @@ class TDengineConnection(object): self._config = kwargs['config'] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) def close(self): """Close current connection. @@ -56,7 +63,8 @@ class TDengineConnection(object): """ if self._conn is None: return None - sub = CTaosInterface.subscribe(self._conn, restart, topic, sql, interval) + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) return TDengineSubscription(sub) def cursor(self): @@ -81,7 +89,8 @@ class TDengineConnection(object): """ pass + if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') conn.close() - print("Hello world") \ No newline at end of file + print("Hello world") diff --git a/src/connector/python/windows/python2/taos/constants.py b/src/connector/python/windows/python2/taos/constants.py index a994bceaf61894ac0bf9a719a574d00a09c584a5..8a8011c3e36c52993e9d03228c2a50e2af6a7c9e 100644 --- a/src/connector/python/windows/python2/taos/constants.py +++ b/src/connector/python/windows/python2/taos/constants.py @@ -3,6 +3,7 @@ from .dbapi import * + class FieldType(object): """TDengine Field Types """ @@ -18,13 +19,21 @@ class FieldType(object): C_BINARY = 8 C_TIMESTAMP = 9 C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index f6fde2619b0a727c74904c2338c813b97a2e602c..5f4666b59396d4dbd7df6daaf62beb814e3b53f7 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -1,10 +1,10 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading # querySeqNum = 0 + class TDengineCursor(object): """Database cursor which is used to manage the context of a fetch operation. @@ -23,7 +23,7 @@ class TDengineCursor(object): if the cursor has not had an operation invoked via the .execute*() method yet. .rowcount:This read-only attribute specifies the number of rows that the last - .execute*() produced (for DQL statements like SELECT) or affected + .execute*() produced (for DQL statements like SELECT) or affected """ def __init__(self, connection=None): @@ -37,7 +37,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection @@ -50,13 +49,14 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow( + self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) self._block_iter = 0 - data = self._block[self._block_iter] + data = self._block[self._block_iter] self._block_iter += 1 return data @@ -91,7 +91,7 @@ class TDengineCursor(object): """ if self._connection is None: return False - + self._reset_result() self._connection = None @@ -106,19 +106,20 @@ class TDengineCursor(object): if not self._connection: # TODO : change the exception raised here raise ProgrammingError("Cursor is not connected") - + self._reset_result() stmt = operation if params is not None: pass - + self._result = CTaosInterface.query(self._connection._conn, stmt) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno == 0: if CTaosInterface.fieldsCount(self._result) == 0: - self._affected_rows += CTaosInterface.affectedRows(self._result) - return CTaosInterface.affectedRows(self._result ) + self._affected_rows += CTaosInterface.affectedRows( + self._result) + return CTaosInterface.affectedRows(self._result) else: self._fields = CTaosInterface.useResult(self._result) return self._handle_result() @@ -138,7 +139,7 @@ class TDengineCursor(object): def fetchmany(self): pass - def fetchall(self): + def fetchall_row(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ if self._result is None or self._fields is None: @@ -147,38 +148,42 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - - def fetchall_block(self): + + def fetchall(self): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) - if num_of_fields == 0: break + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) - return list(map(tuple, zip(*buffer))) - - def nextset(self): """ """ @@ -209,6 +214,7 @@ class TDengineCursor(object): """ self._description = [] for ele in self._fields: - self._description.append((ele['name'], ele['type'], None, None, None, None, False)) - + self._description.append( + (ele['name'], ele['type'], None, None, None, None, False)) + return self._result diff --git a/src/connector/python/windows/python2/taos/dbapi.py b/src/connector/python/windows/python2/taos/dbapi.py index f1c22bdb512224ac712b78b15ec00207587e65c5..594681ada953abf388e503c23199043cf686e1a3 100644 --- a/src/connector/python/windows/python2/taos/dbapi.py +++ b/src/connector/python/windows/python2/taos/dbapi.py @@ -4,6 +4,7 @@ import time import datetime + class DBAPITypeObject(object): def __init__(self, *values): self.values = values @@ -16,23 +17,28 @@ class DBAPITypeObject(object): else: return -1 + Date = datetime.date Time = datetime.time Timestamp = datetime.datetime + def DataFromTicks(ticks): return Date(*time.localtime(ticks)[:3]) + def TimeFromTicks(ticks): return Time(*time.localtime(ticks)[3:6]) + def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) + Binary = bytes # STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) # BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) # NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) # DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() \ No newline at end of file +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/windows/python2/taos/error.py b/src/connector/python/windows/python2/taos/error.py index 24508a72ed78bb6231187bb6de34d57182e31b22..c584badce8320cd35dc81e8f6b613c56163b1a29 100644 --- a/src/connector/python/windows/python2/taos/error.py +++ b/src/connector/python/windows/python2/taos/error.py @@ -1,35 +1,41 @@ """Python exceptions """ + class Error(Exception): def __init__(self, msg=None, errno=None): self.msg = msg self._full_msg = self.msg self.errno = errno - + def __str__(self): return self._full_msg + class Warning(Exception): """Exception raised for important warnings like data truncations while inserting. """ pass + class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. + """Exception raised for errors that are related to the database interface rather than the database itself. """ pass + class DatabaseError(Error): - """Exception raised for errors that are related to the database. + """Exception raised for errors that are related to the database. """ pass + class DataError(DatabaseError): """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. """ pass + class OperationalError(DatabaseError): """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer """ @@ -41,17 +47,20 @@ class IntegrityError(DatabaseError): """ pass + class InternalError(DatabaseError): """Exception raised when the database encounters an internal error. """ pass + class ProgrammingError(DatabaseError): """Exception raised for programming errors. """ pass + class NotSupportedError(DatabaseError): """Exception raised in case a method or database API was used which is not supported by the database,. """ - pass \ No newline at end of file + pass diff --git a/src/connector/python/windows/python2/taos/subscription.py b/src/connector/python/windows/python2/taos/subscription.py index d3cf10d5ada578687689b94454378dd543368e3e..270d9de09217fc58a389981a3542698dd1c0428a 100644 --- a/src/connector/python/windows/python2/taos/subscription.py +++ b/src/connector/python/windows/python2/taos/subscription.py @@ -1,32 +1,33 @@ from .cinterface import CTaosInterface from .error import * + class TDengineSubscription(object): """TDengine subscription object """ + def __init__(self, sub): self._sub = sub - def consume(self): """Consume rows of a subscription """ if self._sub is None: raise OperationalError("Invalid use of consume") - + result, fields = CTaosInterface.consume(self._sub) buffer = [[] for i in range(len(fields))] while True: block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: break + if num_of_fields == 0: + break for i in range(len(fields)): buffer[i].extend(block[i]) self.fields = fields return list(map(tuple, zip(*buffer))) - - def close(self, keepProgress = True): + def close(self, keepProgress=True): """Close the Subscription. """ if self._sub is None: @@ -38,15 +39,19 @@ class TDengineSubscription(object): if __name__ == '__main__': from .connection import TDengineConnection - conn = TDengineConnection(host="127.0.0.1", user="root", password="taosdata", database="test") + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) - for i in range(0,10): + for i in range(0, 10): data = sub.consume() for d in data: print(d) sub.close() - conn.close() \ No newline at end of file + conn.close() diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py index ffed304c855fda28287735d0daab27e17b0e976e..cdcec62a218e72adbda949c0fd666b5ec4abc340 100644 --- a/src/connector/python/windows/python3/setup.py +++ b/src/connector/python/windows/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.3", + version="2.0.7", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python3/taos/__init__.py b/src/connector/python/windows/python3/taos/__init__.py index c6dd929a6a5110300ecd2e1042d48a5ad6a4bf46..b57e25fd2c320956e46b190d9f0a1139db1cced0 100644 --- a/src/connector/python/windows/python3/taos/__init__.py +++ b/src/connector/python/windows/python3/taos/__init__.py @@ -3,12 +3,12 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' __all__ = ['connection', 'cursor'] + def connect(*args, **kwargs): """ Function to return a TDengine connector object @@ -21,4 +21,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) \ No newline at end of file + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index 42b820ca80feda4a269e5e80f288bd1f5e87adcd..ec72474df93a4d13aa13256611044c5bc140f2d3 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -4,11 +4,14 @@ from .error import * import math import datetime + def _convert_millisecond_to_datetime(milli): - return datetime.datetime.fromtimestamp(milli/1000.0) + return datetime.datetime.fromtimestamp(milli / 1000.0) + def _convert_microsecond_to_datetime(micro): - return datetime.datetime.fromtimestamp(micro/1000000.0) + return datetime.datetime.fromtimestamp(micro / 1000000.0) + def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row @@ -18,170 +21,309 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) else: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) + return list(map(_timestamp_converter, ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)])) + def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_byte))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_bool))[ + :abs(num_of_rows)]] + def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] - + return [None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)]] + + +def _crow_tinyint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C tinyint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_TINYINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ubyte))[ + :abs(num_of_rows)]] + + def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_short))[ + :abs(num_of_rows)]] + + +def _crow_smallint_unsigned_to_python( + data, num_of_rows, nbytes=None, micro=False): + """Function to convert C smallint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_SMALLINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_ushort))[ + :abs(num_of_rows)]] + def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + else: + return [None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)]] + + +def _crow_int_unsigned_to_python(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C int row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] + return [ + None if ele == FieldType.C_INT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint))[ + :abs(num_of_rows)]] + def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] else: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] + return [None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_int64))[:abs(num_of_rows)]] + + +def _crow_bigint_unsigned_to_python( + data, + num_of_rows, + nbytes=None, + micro=False): + """Function to convert C bigint row to python row + """ + if num_of_rows > 0: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] + else: + return [ + None if ele == FieldType.C_BIGINT_UNSIGNED_NULL else ele for ele in ctypes.cast( + data, ctypes.POINTER( + ctypes.c_uint64))[ + :abs(num_of_rows)]] + def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)]] + def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] else: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] + return [None if math.isnan(ele) else ele for ele in ctypes.cast( + data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)]] + def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + return [None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode( + 'utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] + def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res = [] - for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: tmpstr = ctypes.c_char_p(data) - res.append( tmpstr.value.decode() ) + res.append(tmpstr.value.decode()) else: - res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res - + + def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows > 0: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode()[0:rbyte] ) + rbyte = ctypes.cast( + data + nbytes * i, + ctypes.POINTER( + ctypes.c_short))[ + :1].pop() + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()[0:rbyte]) except ValueError: res.append(None) return res + def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - res=[] + res = [] if num_of_rows >= 0: for i in range(abs(num_of_rows)): try: - tmpstr = ctypes.c_char_p(data+nbytes*i+2) - res.append( tmpstr.value.decode() ) + tmpstr = ctypes.c_char_p(data + nbytes * i + 2) + res.append(tmpstr.value.decode()) except ValueError: res.append(None) else: for i in range(abs(num_of_rows)): try: - res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + res.append((ctypes.cast(data + nbytes * i + 2, + ctypes.POINTER(ctypes.c_wchar * (nbytes // 4))))[0].value) except ValueError: res.append(None) return res + _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } _CONVERT_FUNC_BLOCK = { FieldType.C_BOOL: _crow_bool_to_python, - FieldType.C_TINYINT : _crow_tinyint_to_python, - FieldType.C_SMALLINT : _crow_smallint_to_python, - FieldType.C_INT : _crow_int_to_python, - FieldType.C_BIGINT : _crow_bigint_to_python, - FieldType.C_FLOAT : _crow_float_to_python, - FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_TINYINT: _crow_tinyint_to_python, + FieldType.C_SMALLINT: _crow_smallint_to_python, + FieldType.C_INT: _crow_int_to_python, + FieldType.C_BIGINT: _crow_bigint_to_python, + FieldType.C_FLOAT: _crow_float_to_python, + FieldType.C_DOUBLE: _crow_double_to_python, FieldType.C_BINARY: _crow_binary_to_python_block, - FieldType.C_TIMESTAMP : _crow_timestamp_to_python, - FieldType.C_NCHAR : _crow_nchar_to_python_block + FieldType.C_TIMESTAMP: _crow_timestamp_to_python, + FieldType.C_NCHAR: _crow_nchar_to_python_block, + FieldType.C_TINYINT_UNSIGNED: _crow_tinyint_unsigned_to_python, + FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, + FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, + FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python } # Corresponding TAOS_FIELD structure in C + + class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), ('type', ctypes.c_char), ('bytes', ctypes.c_short)] # C interface class + + class CTaosInterface(object): libtaos = ctypes.windll.LoadLibrary('taos') @@ -218,7 +360,7 @@ class CTaosInterface(object): except AttributeError: raise AttributeError("config is expected as a str") - if config != None: + if config is not None: CTaosInterface.libtaos.taos_options(3, self._config) CTaosInterface.libtaos.taos_init() @@ -229,7 +371,13 @@ class CTaosInterface(object): """ return self._config - def connect(self, host=None, user="root", password="taosdata", db=None, port=0): + def connect( + self, + host=None, + user="root", + password="taosdata", + db=None, + port=0): ''' Function to connect to server @@ -238,7 +386,7 @@ class CTaosInterface(object): # host try: _host = ctypes.c_char_p(host.encode( - "utf-8")) if host != None else ctypes.c_char_p(None) + "utf-8")) if host is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("host is expected as a str") @@ -257,7 +405,7 @@ class CTaosInterface(object): # db try: _db = ctypes.c_char_p( - db.encode("utf-8")) if db != None else ctypes.c_char_p(None) + db.encode("utf-8")) if db is not None else ctypes.c_char_p(None) except AttributeError: raise AttributeError("db is expected as a str") @@ -270,11 +418,11 @@ class CTaosInterface(object): connection = ctypes.c_void_p(CTaosInterface.libtaos.taos_connect( _host, _user, _password, _db, _port)) - if connection.value == None: + if connection.value is None: print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - #else: + # else: # print('connect to TDengine success') return connection @@ -295,7 +443,8 @@ class CTaosInterface(object): @rtype: 0 on success and -1 on failure ''' try: - return CTaosInterface.libtaos.taos_query(connection, ctypes.c_char_p(sql.encode('utf-8'))) + return CTaosInterface.libtaos.taos_query( + connection, ctypes.c_char_p(sql.encode('utf-8'))) except AttributeError: raise AttributeError("sql is expected as a string") # finally: @@ -310,7 +459,7 @@ class CTaosInterface(object): @staticmethod def subscribe(connection, restart, topic, sql, interval): """Create a subscription - @restart boolean, + @restart boolean, @sql string, sql statement for data query, must be a 'select' statement. @topic string, name of this subscription """ @@ -362,35 +511,49 @@ class CTaosInterface(object): result, ctypes.byref(pblock)) if num_of_rows == 0: return None, 0 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: raise DatabaseError("Invalid data type returned from database") - blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) return blocks, abs(num_of_rows) + @staticmethod def fetchRow(result, fields): pblock = ctypes.c_void_p(0) - pblock = CTaosInterface.libtaos.taos_fetch_row(result) - if pblock : + pblock = CTaosInterface.libtaos.taos_fetch_row(result) + if pblock: num_of_rows = 1 - isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + isMicro = (CTaosInterface.libtaos.taos_result_precision( + result) == FieldType.C_TIMESTAMP_MICRO) blocks = [None] * len(fields) fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + fieldLen = [ + ele for ele in ctypes.cast( + fieldL, ctypes.POINTER( + ctypes.c_int))[ + :len(fields)]] for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] if fields[i]['type'] not in _CONVERT_FUNC: - raise DatabaseError("Invalid data type returned from database") + raise DatabaseError( + "Invalid data type returned from database") if data is None: blocks[i] = [None] else: - blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + blocks[i] = _CONVERT_FUNC[fields[i]['type']]( + data, num_of_rows, fieldLen[i], isMicro) else: return None, 0 return blocks, abs(num_of_rows) @@ -437,7 +600,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_INT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BIGINT): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_FLOAT): # return ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_DOUBLE): @@ -445,7 +608,7 @@ class CTaosInterface(object): # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_BINARY): # return (ctypes.cast(data, ctypes.POINTER(ctypes.c_char))[0:byte]).rstrip('\x00') # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_TIMESTAMP): - # return ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[0] + # return ctypes.cast(data, ctypes.POINTER(ctypes.c_int64))[0] # elif (dtype == CTaosInterface.TSDB_DATA_TYPE_NCHAR): # return (ctypes.cast(data, ctypes.c_char_p).value).rstrip('\x00') @@ -476,4 +639,4 @@ if __name__ == '__main__': print(data) cinter.freeResult(result) - cinter.close(conn) \ No newline at end of file + cinter.close(conn) diff --git a/src/connector/python/windows/python3/taos/connection.py b/src/connector/python/windows/python3/taos/connection.py index d9576a553b810a975429b2cefc03e5e60f240a88..5729d01c6df8c0e58086726c4001467811e9fee5 100644 --- a/src/connector/python/windows/python3/taos/connection.py +++ b/src/connector/python/windows/python3/taos/connection.py @@ -2,9 +2,11 @@ from .cursor import TDengineCursor from .subscription import TDengineSubscription from .cinterface import CTaosInterface + class TDengineConnection(object): """ TDengine connection object """ + def __init__(self, *args, **kwargs): self._conn = None self._host = None @@ -30,7 +32,7 @@ class TDengineConnection(object): # password if 'password' in kwargs: self._password = kwargs['password'] - + # database if 'database' in kwargs: self._database = kwargs['database'] @@ -44,7 +46,12 @@ class TDengineConnection(object): self._config = kwargs['config'] self._chandle = CTaosInterface(self._config) - self._conn = self._chandle.connect(self._host, self._user, self._password, self._database, self._port) + self._conn = self._chandle.connect( + self._host, + self._user, + self._password, + self._database, + self._port) def close(self): """Close current connection. @@ -56,7 +63,8 @@ class TDengineConnection(object): """ if self._conn is None: return None - sub = CTaosInterface.subscribe(self._conn, restart, topic, sql, interval) + sub = CTaosInterface.subscribe( + self._conn, restart, topic, sql, interval) return TDengineSubscription(sub) def cursor(self): @@ -81,7 +89,8 @@ class TDengineConnection(object): """ pass + if __name__ == "__main__": conn = TDengineConnection(host='192.168.1.107') conn.close() - print("Hello world") \ No newline at end of file + print("Hello world") diff --git a/src/connector/python/windows/python3/taos/constants.py b/src/connector/python/windows/python3/taos/constants.py index def2bbc0a8c063e85214a634b60c5db7a5fd1259..49fc17b2fb98a6684e74e4a044651fdc6237518e 100644 --- a/src/connector/python/windows/python3/taos/constants.py +++ b/src/connector/python/windows/python3/taos/constants.py @@ -3,6 +3,7 @@ from .dbapi import * + class FieldType(object): """TDengine Field Types """ @@ -18,13 +19,21 @@ class FieldType(object): C_BINARY = 8 C_TIMESTAMP = 9 C_NCHAR = 10 + C_TINYINT_UNSIGNED = 11 + C_SMALLINT_UNSIGNED = 12 + C_INT_UNSIGNED = 13 + C_BIGINT_UNSIGNED = 14 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 C_TINYINT_NULL = -128 + C_TINYINT_UNSIGNED_NULL = 255 C_SMALLINT_NULL = -32768 + C_SMALLINT_UNSIGNED_NULL = 65535 C_INT_NULL = -2147483648 + C_INT_UNSIGNED_NULL = 4294967295 C_BIGINT_NULL = -9223372036854775808 + C_BIGINT_UNSIGNED_NULL = 18446744073709551615 C_FLOAT_NULL = float('nan') C_DOUBLE_NULL = float('nan') C_BINARY_NULL = bytearray([int('0xff', 16)]) diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index db66b99d3bfe5935a3e5b04c3692453092bcf24e..136cd42fe47a51a34b301730745d73e893861f7c 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -1,7 +1,6 @@ from .cinterface import CTaosInterface from .error import * from .constants import FieldType -import threading # querySeqNum = 0 @@ -24,7 +23,7 @@ class TDengineCursor(object): if the cursor has not had an operation invoked via the .execute*() method yet. .rowcount:This read-only attribute specifies the number of rows that the last - .execute*() produced (for DQL statements like SELECT) or affected + .execute*() produced (for DQL statements like SELECT) or affected """ def __init__(self, connection=None): @@ -38,7 +37,6 @@ class TDengineCursor(object): self._block_iter = 0 self._affected_rows = 0 self._logfile = "" - self._threadId = threading.get_ident() if connection is not None: self._connection = connection @@ -51,13 +49,14 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow( + self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) self._block_iter = 0 - data = self._block[self._block_iter] + data = self._block[self._block_iter] self._block_iter += 1 return data @@ -92,7 +91,7 @@ class TDengineCursor(object): """ if self._connection is None: return False - + self._reset_result() self._connection = None @@ -107,24 +106,25 @@ class TDengineCursor(object): if not self._connection: # TODO : change the exception raised here raise ProgrammingError("Cursor is not connected") - + self._reset_result() stmt = operation if params is not None: pass - + self._result = CTaosInterface.query(self._connection._conn, stmt) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno == 0: if CTaosInterface.fieldsCount(self._result) == 0: - self._affected_rows += CTaosInterface.affectedRows(self._result ) - return CTaosInterface.affectedRows(self._result ) + self._affected_rows += CTaosInterface.affectedRows( + self._result) + return CTaosInterface.affectedRows(self._result) else: - self._fields = CTaosInterface.useResult(self._result ) + self._fields = CTaosInterface.useResult(self._result) return self._handle_result() else: - raise ProgrammingError(CTaosInterface.errStr(self._result ), errno) + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) def executemany(self, operation, seq_of_parameters): """Prepare a database operation (query or command) and then execute it against all parameter sequences or mappings found in the sequence seq_of_parameters. @@ -139,7 +139,7 @@ class TDengineCursor(object): def fetchmany(self): pass - def fetchall(self): + def fetchall_row(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ if self._result is None or self._fields is None: @@ -148,10 +148,13 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields @@ -159,27 +162,28 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def fetchall_block(self): + def fetchall(self): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock( + self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: - raise ProgrammingError(CTaosInterface.errStr(self._result), errno) - if num_of_fields == 0: break + raise ProgrammingError( + CTaosInterface.errStr( + self._result), errno) + if num_of_fields == 0: + break self._rowcount += num_of_fields for i in range(len(self._fields)): buffer[i].extend(block[i]) - return list(map(tuple, zip(*buffer))) - - def nextset(self): """ """ @@ -204,12 +208,13 @@ class TDengineCursor(object): self._block_rows = -1 self._block_iter = 0 self._affected_rows = 0 - + def _handle_result(self): """Handle the return result from query. """ self._description = [] for ele in self._fields: - self._description.append((ele['name'], ele['type'], None, None, None, None, False)) - + self._description.append( + (ele['name'], ele['type'], None, None, None, None, False)) + return self._result diff --git a/src/connector/python/windows/python3/taos/dbapi.py b/src/connector/python/windows/python3/taos/dbapi.py index 9b1cb1321c14619782c801e9381010f1f67fbc2e..a29621f7a3594a618b59b30bdc96197c4222a619 100644 --- a/src/connector/python/windows/python3/taos/dbapi.py +++ b/src/connector/python/windows/python3/taos/dbapi.py @@ -4,6 +4,7 @@ import time import datetime + class DBAPITypeObject(object): def __init__(self, *values): self.values = values @@ -16,23 +17,28 @@ class DBAPITypeObject(object): else: return -1 + Date = datetime.date Time = datetime.time Timestamp = datetime.datetime + def DataFromTicks(ticks): return Date(*time.localtime(ticks)[:3]) + def TimeFromTicks(ticks): return Time(*time.localtime(ticks)[3:6]) + def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) + Binary = bytes # STRING = DBAPITypeObject(*constants.FieldType.get_string_types()) # BINARY = DBAPITypeObject(*constants.FieldType.get_binary_types()) # NUMBER = BAPITypeObject(*constants.FieldType.get_number_types()) # DATETIME = DBAPITypeObject(*constants.FieldType.get_timestamp_types()) -# ROWID = DBAPITypeObject() \ No newline at end of file +# ROWID = DBAPITypeObject() diff --git a/src/connector/python/windows/python3/taos/error.py b/src/connector/python/windows/python3/taos/error.py index ccc0e61d84597a7cae9b360b7fa2434b9bc47401..238b293a0b609570e7b5d536648c6ada3ca2f209 100644 --- a/src/connector/python/windows/python3/taos/error.py +++ b/src/connector/python/windows/python3/taos/error.py @@ -1,35 +1,41 @@ """Python exceptions """ + class Error(Exception): def __init__(self, msg=None, errno=None): self.msg = msg self._full_msg = self.msg self.errno = errno - + def __str__(self): return self._full_msg + class Warning(Exception): """Exception raised for important warnings like data truncations while inserting. """ pass + class InterfaceError(Error): - """Exception raised for errors that are related to the database interface rather than the database itself. + """Exception raised for errors that are related to the database interface rather than the database itself. """ pass + class DatabaseError(Error): - """Exception raised for errors that are related to the database. + """Exception raised for errors that are related to the database. """ pass + class DataError(DatabaseError): """Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range. """ pass + class OperationalError(DatabaseError): """Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer """ @@ -41,17 +47,20 @@ class IntegrityError(DatabaseError): """ pass + class InternalError(DatabaseError): """Exception raised when the database encounters an internal error. """ pass + class ProgrammingError(DatabaseError): """Exception raised for programming errors. """ pass + class NotSupportedError(DatabaseError): """Exception raised in case a method or database API was used which is not supported by the database,. """ - pass \ No newline at end of file + pass diff --git a/src/connector/python/windows/python3/taos/subscription.py b/src/connector/python/windows/python3/taos/subscription.py index d3cf10d5ada578687689b94454378dd543368e3e..270d9de09217fc58a389981a3542698dd1c0428a 100644 --- a/src/connector/python/windows/python3/taos/subscription.py +++ b/src/connector/python/windows/python3/taos/subscription.py @@ -1,32 +1,33 @@ from .cinterface import CTaosInterface from .error import * + class TDengineSubscription(object): """TDengine subscription object """ + def __init__(self, sub): self._sub = sub - def consume(self): """Consume rows of a subscription """ if self._sub is None: raise OperationalError("Invalid use of consume") - + result, fields = CTaosInterface.consume(self._sub) buffer = [[] for i in range(len(fields))] while True: block, num_of_fields = CTaosInterface.fetchBlock(result, fields) - if num_of_fields == 0: break + if num_of_fields == 0: + break for i in range(len(fields)): buffer[i].extend(block[i]) self.fields = fields return list(map(tuple, zip(*buffer))) - - def close(self, keepProgress = True): + def close(self, keepProgress=True): """Close the Subscription. """ if self._sub is None: @@ -38,15 +39,19 @@ class TDengineSubscription(object): if __name__ == '__main__': from .connection import TDengineConnection - conn = TDengineConnection(host="127.0.0.1", user="root", password="taosdata", database="test") + conn = TDengineConnection( + host="127.0.0.1", + user="root", + password="taosdata", + database="test") # Generate a cursor object to run SQL commands sub = conn.subscribe(True, "test", "select * from meters;", 1000) - for i in range(0,10): + for i in range(0, 10): data = sub.consume() for d in data: print(d) sub.close() - conn.close() \ No newline at end of file + conn.close() diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index 9da831c9c137a0551306d343dd112189816ead5b..e9ed2996c74e2c59d56245e6fc1e932ebb07dfb0 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) @@ -6,12 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX) - ADD_LIBRARY(tcq ${SRC}) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(tcq tutil common taos_static) - ELSE () - TARGET_LINK_LIBRARIES(tcq tutil common taos) - ENDIF () - ADD_SUBDIRECTORY(test) +ADD_LIBRARY(tcq ${SRC}) +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(tcq tutil common taos_static) +ELSE () + TARGET_LINK_LIBRARIES(tcq tutil common taos) ENDIF () +ADD_SUBDIRECTORY(test) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 0fe5ea78d46e6d2796c1d71531c87ec44cb8ee0d..fb0c1508cb051c577fed9bde534d55e297822fe2 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -50,10 +50,13 @@ typedef struct { void *dbConn; void *tmrCtrl; pthread_mutex_t mutex; + int32_t delete; + int32_t cqObjNum; } SCqContext; typedef struct SCqObj { tmr_h tmrId; + int64_t rid; uint64_t uid; int32_t tid; // table ID int32_t rowSize; // bytes of a row @@ -69,6 +72,90 @@ typedef struct SCqObj { static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); +int32_t cqObjRef = -1; + +void cqRmFromList(SCqObj *pObj) { + //LOCK in caller + + SCqContext *pContext = pObj->pContext; + + if (pObj->prev) { + pObj->prev->next = pObj->next; + } else { + pContext->pHead = pObj->next; + } + + if (pObj->next) { + pObj->next->prev = pObj->prev; + } + +} + +static void freeSCqContext(void *handle) { + if (handle == NULL) { + return; + } + SCqContext *pContext = handle; + pthread_mutex_destroy(&pContext->mutex); + + taosTmrCleanUp(pContext->tmrCtrl); + pContext->tmrCtrl = NULL; + cDebug("vgId:%d, CQ is closed", pContext->vgId); + free(pContext); +} + + +void cqFree(void *handle) { + if (tsEnableStream == 0) { + return; + } + SCqObj *pObj = handle; + SCqContext *pContext = pObj->pContext; + int32_t delete = 0; + + pthread_mutex_lock(&pContext->mutex); + + // free the resources associated + if (pObj->pStream) { + taos_close_stream(pObj->pStream); + pObj->pStream = NULL; + } else { + taosTmrStop(pObj->tmrId); + pObj->tmrId = 0; + } + + cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); + tdFreeSchema(pObj->pSchema); + free(pObj->dstTable); + free(pObj->sqlStr); + free(pObj); + + pContext->cqObjNum--; + + if (pContext->cqObjNum <= 0 && pContext->delete) { + delete = 1; + } + + pthread_mutex_unlock(&pContext->mutex); + + if (delete) { + freeSCqContext(pContext); + } +} + + +void cqCreateRef() { + int32_t ref = atomic_load_32(&cqObjRef); + if (ref == -1) { + ref = taosOpenRef(4096, cqFree); + + if (atomic_val_compare_exchange_32(&cqObjRef, -1, ref) != -1) { + taosCloseRef(ref); + } + } +} + + void *cqOpen(void *ahandle, const SCqCfg *pCfg) { if (tsEnableStream == 0) { return NULL; @@ -79,6 +166,8 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { return NULL; } + cqCreateRef(); + pContext->tmrCtrl = taosTmrInit(0, 0, 0, "CQ"); tstrncpy(pContext->user, pCfg->user, sizeof(pContext->user)); @@ -97,11 +186,13 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { pthread_mutex_init(&pContext->mutex, NULL); + cDebug("vgId:%d, CQ is opened", pContext->vgId); return pContext; } + void cqClose(void *handle) { if (tsEnableStream == 0) { return; @@ -109,30 +200,46 @@ void cqClose(void *handle) { SCqContext *pContext = handle; if (handle == NULL) return; + pContext->delete = 1; + int32_t hasCq = 0; + int32_t existLoop = 0; + // stop all CQs cqStop(pContext); - // free all resources - pthread_mutex_lock(&pContext->mutex); + int64_t rid = 0; - SCqObj *pObj = pContext->pHead; - while (pObj) { - SCqObj *pTemp = pObj; - pObj = pObj->next; - tdFreeSchema(pTemp->pSchema); - tfree(pTemp->sqlStr); - free(pTemp); - } - - pthread_mutex_unlock(&pContext->mutex); + while (1) { + pthread_mutex_lock(&pContext->mutex); - pthread_mutex_destroy(&pContext->mutex); + SCqObj *pObj = pContext->pHead; + if (pObj) { + cqRmFromList(pObj); - taosTmrCleanUp(pContext->tmrCtrl); - pContext->tmrCtrl = NULL; + rid = pObj->rid; - cDebug("vgId:%d, CQ is closed", pContext->vgId); - free(pContext); + hasCq = 1; + + if (pContext->pHead == NULL) { + existLoop = 1; + } + } else { + pthread_mutex_unlock(&pContext->mutex); + break; + } + + pthread_mutex_unlock(&pContext->mutex); + + taosRemoveRef(cqObjRef, rid); + + if (existLoop) { + break; + } + } + + if (hasCq == 0) { + freeSCqContext(pContext); + } } void cqStart(void *handle) { @@ -160,6 +267,7 @@ void cqStop(void *handle) { if (tsEnableStream == 0) { return; } + SCqContext *pContext = handle; cDebug("vgId:%d, stop all CQs", pContext->vgId); if (pContext->dbConn == NULL || pContext->master == 0) return; @@ -191,7 +299,8 @@ void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, ch return NULL; } SCqContext *pContext = handle; - + int64_t rid = 0; + SCqObj *pObj = calloc(sizeof(SCqObj), 1); if (pObj == NULL) return NULL; @@ -213,32 +322,36 @@ void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, ch if (pContext->pHead) pContext->pHead->prev = pObj; pContext->pHead = pObj; + pContext->cqObjNum++; + + pObj->rid = taosAddRef(cqObjRef, pObj); + cqCreateStream(pContext, pObj); + rid = pObj->rid; + pthread_mutex_unlock(&pContext->mutex); - return pObj; + + return (void *)rid; } void cqDrop(void *handle) { if (tsEnableStream == 0) { return; } - SCqObj *pObj = handle; - SCqContext *pContext = pObj->pContext; - - pthread_mutex_lock(&pContext->mutex); - if (pObj->prev) { - pObj->prev->next = pObj->next; - } else { - pContext->pHead = pObj->next; - } - - if (pObj->next) { - pObj->next->prev = pObj->prev; + SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)handle); + if (pObj == NULL) { + return; } + + SCqContext *pContext = pObj->pContext; + + pthread_mutex_lock(&pContext->mutex); + cqRmFromList(pObj); + // free the resources associated if (pObj->pStream) { taos_close_stream(pObj->pStream); @@ -248,17 +361,18 @@ void cqDrop(void *handle) { pObj->tmrId = 0; } - cInfo("vgId:%d, id:%d CQ:%s is dropped", pContext->vgId, pObj->tid, pObj->sqlStr); - tdFreeSchema(pObj->pSchema); - free(pObj->dstTable); - free(pObj->sqlStr); - free(pObj); - pthread_mutex_unlock(&pContext->mutex); + + taosRemoveRef(cqObjRef, (int64_t)handle); + taosReleaseRef(cqObjRef, (int64_t)handle); } static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { - SCqObj* pObj = (SCqObj*)param; + SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); + if (pObj == NULL) { + return; + } + SCqContext* pContext = pObj->pContext; SSqlObj* pSql = (SSqlObj*)result; if (atomic_val_compare_exchange_ptr(&(pContext->dbConn), NULL, pSql->pTscObj) != NULL) { @@ -267,10 +381,16 @@ static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { pthread_mutex_lock(&pContext->mutex); cqCreateStream(pContext, pObj); pthread_mutex_unlock(&pContext->mutex); + + taosReleaseRef(cqObjRef, (int64_t)param); } static void cqProcessCreateTimer(void *param, void *tmrId) { - SCqObj* pObj = (SCqObj*)param; + SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); + if (pObj == NULL) { + return; + } + SCqContext* pContext = pObj->pContext; if (pContext->dbConn == NULL) { @@ -281,6 +401,8 @@ static void cqProcessCreateTimer(void *param, void *tmrId) { cqCreateStream(pContext, pObj); pthread_mutex_unlock(&pContext->mutex); } + + taosReleaseRef(cqObjRef, (int64_t)param); } static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { @@ -288,13 +410,13 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { if (pContext->dbConn == NULL) { cDebug("vgId:%d, create dbConn after 1000 ms", pContext->vgId); - pObj->tmrId = taosTmrStart(cqProcessCreateTimer, 1000, pObj, pContext->tmrCtrl); + pObj->tmrId = taosTmrStart(cqProcessCreateTimer, 1000, (void *)pObj->rid, pContext->tmrCtrl); return; } pObj->tmrId = 0; if (pObj->pStream == NULL) { - pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL); + pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, (void *)pObj->rid, NULL); // TODO the pObj->pStream may be released if error happens if (pObj->pStream) { @@ -308,18 +430,28 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { } static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { - SCqObj *pObj = (SCqObj *)param; + SCqObj* pObj = (SCqObj*)taosAcquireRef(cqObjRef, (int64_t)param); + if (pObj == NULL) { + return; + } + if (tres == NULL && row == NULL) { taos_close_stream(pObj->pStream); pObj->pStream = NULL; + + taosReleaseRef(cqObjRef, (int64_t)param); + return; } SCqContext *pContext = pObj->pContext; STSchema *pSchema = pObj->pSchema; - if (pObj->pStream == NULL) return; - + if (pObj->pStream == NULL) { + taosReleaseRef(cqObjRef, (int64_t)param); + return; + } + cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; @@ -343,7 +475,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { char buf[TSDB_MAX_NCHAR_LEN]; int32_t len = taos_fetch_lengths(tres)[i]; taosMbsToUcs4(val, len, buf, sizeof(buf), &len); - memcpy(val + sizeof(VarDataLenT), buf, len); + memcpy((char *)val + sizeof(VarDataLenT), buf, len); varDataLen(val) = len; } tdAppendColVal(trow, val, c->type, c->bytes, c->offset); @@ -370,5 +502,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { // write into vnode write queue pContext->cqWrite(pContext->vgId, pHead, TAOS_QTYPE_CQ, NULL); free(buffer); + + taosReleaseRef(cqObjRef, (int64_t)param); } diff --git a/src/cq/test/CMakeLists.txt b/src/cq/test/CMakeLists.txt index 87a8705fcd53be733d7ecec840ce38355c00d737..cd124567afd8766173cf07e7a6191ab473be1714 100644 --- a/src/cq/test/CMakeLists.txt +++ b/src/cq/test/CMakeLists.txt @@ -1,6 +1,6 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) LIST(APPEND CQTEST_SRC ./cqtest.c) ADD_EXECUTABLE(cqtest ${CQTEST_SRC}) -TARGET_LINK_LIBRARIES(cqtest tcq) +TARGET_LINK_LIBRARIES(cqtest tcq taos_static) diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 699ca00a250866122023d8a4cd789f91bd038609..dd18f0092024834c8cb0d230b27f5905ea2339df 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) @@ -10,40 +10,47 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) - ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosd taos_static) - ELSE () - TARGET_LINK_LIBRARIES(taosd taos) - ENDIF () - - IF (TD_ACCOUNT) - TARGET_LINK_LIBRARIES(taosd account) - ENDIF () - - IF (TD_GRANT) - TARGET_LINK_LIBRARIES(taosd grant) - ENDIF () - - IF (TD_MQTT) - TARGET_LINK_LIBRARIES(taosd mqtt) - ENDIF () - - SET(PREPARE_ENV_CMD "prepare_env_cmd") - SET(PREPARE_ENV_TARGET "prepare_env_target") - ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} - POST_BUILD - COMMAND echo "make test directory" - DEPENDS taosd - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ - COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ - COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg - COMMENT "prepare taosd environment") - ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) +ADD_EXECUTABLE(taosd ${SRC}) +TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync) + +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosd taos_static) +ELSE () + TARGET_LINK_LIBRARIES(taosd taos) +ENDIF () + +IF (TD_ACCOUNT) + TARGET_LINK_LIBRARIES(taosd account) +ENDIF () + +IF (TD_GRANT) + TARGET_LINK_LIBRARIES(taosd grant) +ENDIF () + +IF (TD_MQTT) + TARGET_LINK_LIBRARIES(taosd mqtt) +ENDIF () + +IF (TD_TOPIC) + TARGET_LINK_LIBRARIES(taosd topic) ENDIF () + +IF (TD_MODULE AND TD_LINUX) + TARGET_LINK_LIBRARIES(taosd module dl) +ENDIF () + +SET(PREPARE_ENV_CMD "prepare_env_cmd") +SET(PREPARE_ENV_TARGET "prepare_env_target") +ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD} + POST_BUILD + COMMAND echo "make test directory" + DEPENDS taosd + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/ + COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/ + COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMAND ${CMAKE_COMMAND} -E echo monitor 0 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg + COMMENT "prepare taosd environment") +ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) diff --git a/src/dnode/src/dnodeCfg.c b/src/dnode/src/dnodeCfg.c index f495dbe285c196f3e32d8a0ace8736c8f44d668d..fd5956b37f28603893b43ca91f8cfd1177dccad2 100644 --- a/src/dnode/src/dnodeCfg.c +++ b/src/dnode/src/dnodeCfg.c @@ -97,7 +97,7 @@ static int32_t dnodeReadCfg() { goto PARSE_CFG_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PARSE_CFG_OVER; @@ -115,7 +115,7 @@ static int32_t dnodeReadCfg() { dError("failed to read %s, dnodeId not found", file); goto PARSE_CFG_OVER; } - cfg.dnodeId = dnodeId->valueint; + cfg.dnodeId = (int32_t)dnodeId->valueint; cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId"); if (!clusterId || clusterId->type != cJSON_String) { @@ -156,7 +156,7 @@ static int32_t dnodeWriteCfg() { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index be26bb967bf7e987e9e990b25fb046f2b8664841..87baff30673afc68eb23a00bef279433a422ba67 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -29,8 +29,8 @@ typedef struct { static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}}; int64_t tsMinFreeMemSizeForStart = 0; -static int bindTcpPort(int port) { - int serverSocket; +static int32_t bindTcpPort(uint16_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { @@ -45,22 +45,22 @@ static int bindTcpPort(int port) { if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { dError("port:%d tcp bind() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } if (listen(serverSocket, 5) < 0) { dError("port:%d listen() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } - close(serverSocket); + taosCloseSocket(serverSocket); return 0; } -static int bindUdpPort(int port) { - int serverSocket; +static int32_t bindUdpPort(int16_t port) { + SOCKET serverSocket; struct sockaddr_in server_addr; if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { @@ -75,19 +75,19 @@ static int bindUdpPort(int port) { if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { dError("port:%d udp bind() fail: %s", port, strerror(errno)); - close(serverSocket); + taosCloseSocket(serverSocket); return -1; } - close(serverSocket); + taosCloseSocket(serverSocket); return 0; } -static int dnodeCheckNetwork() { - int ret; - int startPort = tsServerPort; +static int32_t dnodeCheckNetwork() { + int32_t ret; + uint16_t startPort = tsServerPort; - for (int port = startPort; port < startPort + 12; port++) { + for (uint16_t port = startPort; port < startPort + 12; port++) { ret = bindTcpPort(port); if (0 != ret) { dError("failed to tcp bind port %d, quit", port); @@ -103,7 +103,7 @@ static int dnodeCheckNetwork() { return 0; } -static int dnodeCheckMem() { +static int32_t dnodeCheckMem() { float memoryUsedMB; float memoryAvailMB; if (true != taosGetSysMemory(&memoryUsedMB)) { @@ -121,12 +121,12 @@ static int dnodeCheckMem() { return 0; } -static int dnodeCheckCpu() { +static int32_t dnodeCheckCpu() { // TODO: return 0; } -static int dnodeCheckDisk() { +static int32_t dnodeCheckDisk() { taosGetDisk(); if (tsAvailDataDirGB < tsMinimalDataDirGB) { @@ -147,24 +147,24 @@ static int dnodeCheckDisk() { return 0; } -static int dnodeCheckOs() { +static int32_t dnodeCheckOs() { // TODO: return 0; } -static int dnodeCheckAccess() { +static int32_t dnodeCheckAccess() { // TODO: return 0; } -static int dnodeCheckVersion() { +static int32_t dnodeCheckVersion() { // TODO: return 0; } -static int dnodeCheckDatafile() { +static int32_t dnodeCheckDatafile() { // TODO: return 0; diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c index e7dc7efeb269b0a18056d00e001b15fbf6775d1d..9554651776ad0e9c961a3d15955491b224d869bf 100644 --- a/src/dnode/src/dnodeEps.c +++ b/src/dnode/src/dnodeEps.c @@ -152,7 +152,7 @@ static int32_t dnodeReadEps() { goto PRASE_EPS_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PRASE_EPS_OVER; @@ -199,7 +199,7 @@ static int32_t dnodeReadEps() { dError("failed to read %s, dnodeId not found", file); goto PRASE_EPS_OVER; } - ep->dnodeId = dnodeId->valueint; + ep->dnodeId = (int32_t)dnodeId->valueint; cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn"); if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { @@ -277,7 +277,7 @@ static int32_t dnodeWriteEps() { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index dc89487f8b4c5ef2d2434e815d2ba1d8a0311515..0dca116d84a8267231acba686018cf6c07ae15e0 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -80,7 +80,7 @@ void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) { pthread_mutex_lock(&tsMInfosMutex); dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse); - for (int i = 0; i < ep->numOfEps; ++i) { + for (int32_t i = 0; i < ep->numOfEps; ++i) { ep->port[i] -= TSDB_PORT_DNODEDNODE; dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]); } @@ -108,7 +108,7 @@ void dnodeGetMInfos(SMInfos *pMinfos) { void dnodeGetEpSetForPeer(SRpcEpSet *epSet) { pthread_mutex_lock(&tsMInfosMutex); *epSet = tsMEpSet; - for (int i = 0; i < epSet->numOfEps; ++i) { + for (int32_t i = 0; i < epSet->numOfEps; ++i) { epSet->port[i] += TSDB_PORT_DNODEDNODE; } pthread_mutex_unlock(&tsMInfosMutex); @@ -171,7 +171,7 @@ static int32_t dnodeReadMInfos() { goto PARSE_MINFOS_OVER; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { dError("failed to read %s, content is null", file); goto PARSE_MINFOS_OVER; @@ -189,14 +189,14 @@ static int32_t dnodeReadMInfos() { dError("failed to read mnodeEpSet.json, inUse not found"); goto PARSE_MINFOS_OVER; } - tsMInfos.inUse = inUse->valueint; + tsMInfos.inUse = (int8_t)inUse->valueint; cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { dError("failed to read mnodeEpSet.json, nodeNum not found"); goto PARSE_MINFOS_OVER; } - minfos.mnodeNum = nodeNum->valueint; + minfos.mnodeNum = (int8_t)nodeNum->valueint; cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { @@ -204,13 +204,13 @@ static int32_t dnodeReadMInfos() { goto PARSE_MINFOS_OVER; } - int size = cJSON_GetArraySize(nodeInfos); + int32_t size = cJSON_GetArraySize(nodeInfos); if (size != minfos.mnodeNum) { dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); goto PARSE_MINFOS_OVER; } - for (int i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i); if (nodeInfo == NULL) continue; @@ -227,7 +227,7 @@ static int32_t dnodeReadMInfos() { } SMInfo *pMinfo = &minfos.mnodeInfos[i]; - pMinfo->mnodeId = nodeId->valueint; + pMinfo->mnodeId = (int32_t)nodeId->valueint; tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp); @@ -286,7 +286,7 @@ static int32_t dnodeWriteMInfos() { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 0863666f762e792e495c972b63a754492f2a3a9f..e4942c49aaba48db8d2a1c74a0af532769de9553 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -60,7 +60,7 @@ int32_t dnodeInitMPeer() { void dnodeCleanupMPeer() { for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { SMPeerWorker *pWorker = tsMPeerWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMPeerQset); } dDebug("dnode mpeer worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMPeer() { for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { SMPeerWorker *pWorker = tsMPeerWP.worker + i; dDebug("dnode mpeer worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mpeer worker:%d join success", i); diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 9027c346f57c20d91d943c0df8d061c931240101..90332e6783bc4861928833d5794f0787f80be993 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -40,7 +40,7 @@ static void *dnodeProcessMReadQueue(void *param); int32_t dnodeInitMRead() { tsMReadQset = taosOpenQset(); - tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2; + tsMReadWP.maxNum = (int32_t)(tsNumOfCores * tsNumOfThreadsPerCore / 2); tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum); tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum); tsMReadWP.curNum = 0; @@ -60,7 +60,7 @@ int32_t dnodeInitMRead() { void dnodeCleanupMRead() { for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { SMReadWorker *pWorker = tsMReadWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMReadQset); } dDebug("dnode mread worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMRead() { for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { SMReadWorker *pWorker = tsMReadWP.worker + i; dDebug("dnode mread worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mread worker:%d start to join", i); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 8c9e22ef4b19133863f32140fb6ddae292caf5a9..a409d537fa8a56f03ed79d68358ac70b780e74e9 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -60,7 +60,7 @@ int32_t dnodeInitMWrite() { void dnodeCleanupMWrite() { for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { SMWriteWorker *pWorker = tsMWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsMWriteQset); } dDebug("dnode mwrite worker:%d is closed", i); @@ -69,7 +69,7 @@ void dnodeCleanupMWrite() { for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { SMWriteWorker *pWorker = tsMWriteWP.worker + i; dDebug("dnode mwrite worker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } dDebug("dnode mwrite worker:%d join success", i); @@ -145,6 +145,14 @@ void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) { return; } + dTrace("msg:%p, app:%p type:%s master:%p will be responsed", pWrite, pWrite->rpcMsg.ahandle, + taosMsg[pWrite->rpcMsg.msgType], pWrite->pBatchMasterMsg); + if (pWrite->pBatchMasterMsg && pWrite != pWrite->pBatchMasterMsg) { + dError("msg:%p, app:%p type:%s master:%p sub message should not response!", pWrite, pWrite->rpcMsg.ahandle, + taosMsg[pWrite->rpcMsg.msgType], pWrite->pBatchMasterMsg); + return; + } + SRpcMsg rpcRsp = { .handle = pWrite->rpcMsg.handle, .pCont = pWrite->rpcRsp.rsp, diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 517a9e9bc83fb454bb4e2c43e31cbd8c90ad91cd..410e6bb1888a11858e5d091f3f82c54df4dc2022 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -21,7 +21,7 @@ #include "tconfig.h" #include "tfile.h" #include "twal.h" -// #include "tfs.h" +#include "tfs.h" #include "tsync.h" #include "dnodeStep.h" #include "dnodePeer.h" @@ -39,6 +39,13 @@ #include "dnodeMPeer.h" #include "dnodeShell.h" #include "dnodeTelemetry.h" +#include "module.h" + +#if !defined(_MODULE) || !defined(_TD_LINUX) +int32_t moduleStart() { return 0; } +void moduleStop() {} +#endif + void *tsDnodeTmr = NULL; static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED; @@ -63,13 +70,16 @@ static SStep tsDnodeSteps[] = { {"dnode-vread", dnodeInitVRead, dnodeCleanupVRead}, {"dnode-vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, {"dnode-vmgmt", dnodeInitVMgmt, dnodeCleanupVMgmt}, - {"dnode-mread", dnodeInitMRead, dnodeCleanupMRead}, - {"dnode-mwrite", dnodeInitMWrite, dnodeCleanupMWrite}, - {"dnode-mpeer", dnodeInitMPeer, dnodeCleanupMPeer}, + {"dnode-mread", dnodeInitMRead, NULL}, + {"dnode-mwrite", dnodeInitMWrite, NULL}, + {"dnode-mpeer", dnodeInitMPeer, NULL}, {"dnode-client", dnodeInitClient, dnodeCleanupClient}, {"dnode-server", dnodeInitServer, dnodeCleanupServer}, {"dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes}, {"dnode-modules", dnodeInitModules, dnodeCleanupModules}, + {"dnode-mread", NULL, dnodeCleanupMRead}, + {"dnode-mwrite", NULL, dnodeCleanupMWrite}, + {"dnode-mpeer", NULL, dnodeCleanupMPeer}, {"dnode-shell", dnodeInitShell, dnodeCleanupShell}, {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer}, {"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, @@ -146,6 +156,7 @@ int32_t dnodeInitSystem() { } dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING); + moduleStart(); dnodeReportStep("TDengine", "initialized successfully", 1); dInfo("TDengine is initialized successfully"); @@ -155,6 +166,7 @@ int32_t dnodeInitSystem() { void dnodeCleanUpSystem() { if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) { + moduleStop(); dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED); dnodeCleanupTmr(); dnodeCleanupComponents(); @@ -189,32 +201,54 @@ static void dnodeCheckDataDirOpenned(char *dir) { } static int32_t dnodeInitStorage() { - if (dnodeCreateDir(tsDataDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno)); - return -1; + if (tsDiskCfgNum == 1 && dnodeCreateDir(tsDataDir) < 0) { + dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno)); + return -1; + } + + if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) { + dError("failed to init TFS since %s", tstrerror(terrno)); + return -1; } + strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN); sprintf(tsMnodeDir, "%s/mnode", tsDataDir); sprintf(tsVnodeDir, "%s/vnode", tsDataDir); sprintf(tsDnodeDir, "%s/dnode", tsDataDir); - sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); + // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir); //TODO(dengyihao): no need to init here if (dnodeCreateDir(tsMnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno)); return -1; } - //TODO(dengyihao): no need to init here - if (dnodeCreateDir(tsVnodeDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsVnodeDir, strerror(errno)); - return -1; - } + if (dnodeCreateDir(tsDnodeDir) < 0) { dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno)); return -1; - } - if (dnodeCreateDir(tsVnodeBakDir) < 0) { - dError("failed to create dir: %s, reason: %s", tsVnodeBakDir, strerror(errno)); - return -1; + } + + if (tfsMkdir("vnode") < 0) { + dError("failed to create vnode dir since %s", tstrerror(terrno)); + return -1; + } + + if (tfsMkdir("vnode_bak") < 0) { + dError("failed to create vnode_bak dir since %s", tstrerror(terrno)); + return -1; + } + + TDIR *tdir = tfsOpendir("vnode_bak/.staging"); + bool stagingNotEmpty = tfsReaddir(tdir) != NULL; + tfsClosedir(tdir); + + if (stagingNotEmpty) { + dError("vnode_bak/.staging dir not empty, fix it first."); + return -1; + } + + if (tfsMkdir("vnode_bak/.staging") < 0) { + dError("failed to create vnode_bak/.staging dir since %s", tstrerror(terrno)); + return -1; } dnodeCheckDataDirOpenned(tsDnodeDir); @@ -223,7 +257,7 @@ static int32_t dnodeInitStorage() { return 0; } -static void dnodeCleanupStorage() {} +static void dnodeCleanupStorage() { tfsDestroy(); } bool dnodeIsFirstDeploy() { return strcmp(tsFirst, tsLocalEp) == 0; diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 5ee10abc30311792bcd68e579567eea74fa93821..b8ce1c802b2475a8aab76a6bb7f0264d2c9b39a0 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -43,6 +43,7 @@ int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeDispatchToVMgmtQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeDispatchToVMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToVMgmtQueue; @@ -90,7 +91,10 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { }; if (pMsg->pCont == NULL) return; - if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) return dnodeSendStartupStep(pMsg); + if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) { + dnodeSendStartupStep(pMsg); + return; + } if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { rspMsg.code = TSDB_CODE_APP_NOT_READY; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 79cc70005b6e83bc1c455abb7cd3709ae29a8643..50343cfd32405d5408cc9d6c5e939db30bf9b832 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -36,19 +36,23 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_QUERY] = dnodeDispatchToVReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_FETCH] = dnodeDispatchToVReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue; - + // the following message shall be treated as mnode write dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue; @@ -57,20 +61,20 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue; - - // the following message shall be treated as mnode query + + // the following message shall be treated as mnode query dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue; dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep; - int32_t numOfThreads = (tsNumOfCores * tsNumOfThreadsPerCore) / 2.0; + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } @@ -153,7 +157,7 @@ static int32_t dnodeAuthNettestUser(char *user, char *spi, char *encrypt, char * } static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0; + if (dnodeAuthNettestUser(user, spi, encrypt, secret, ckey) == 0) return 0; int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); if (code != TSDB_CODE_APP_NOT_READY) return code; @@ -164,7 +168,7 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char rpcMsg.pCont = pMsg; rpcMsg.contLen = sizeof(SAuthMsg); rpcMsg.msgType = TSDB_MSG_TYPE_DM_AUTH; - + dDebug("user:%s, send auth msg to mnodes", user); SRpcMsg rpcRsp = {0}; dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); @@ -202,21 +206,21 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { SRpcMsg rpcRsp = {0}; dnodeSendMsgToMnodeRecv(&rpcMsg, &rpcRsp); terrno = rpcRsp.code; - + if (rpcRsp.code != 0) { rpcFreeCont(rpcRsp.pCont); dError("vgId:%d, tid:%d failed to config table from mnode", vgId, tid); return NULL; } else { dInfo("vgId:%d, tid:%d config table msg is received", vgId, tid); - + // delete this after debug finished SMDCreateTableMsg *pTable = rpcRsp.pCont; int16_t numOfColumns = htons(pTable->numOfColumns); int16_t numOfTags = htons(pTable->numOfTags); int32_t tableId = htonl(pTable->tid); uint64_t uid = htobe64(pTable->uid); - dInfo("table:%s, numOfColumns:%d numOfTags:%d tid:%d uid:%" PRIu64, pTable->tableId, numOfColumns, numOfTags, tableId, uid); + dInfo("table:%s, numOfColumns:%d numOfTags:%d tid:%d uid:%" PRIu64, pTable->tableFname, numOfColumns, numOfTags, tableId, uid); return rpcRsp.pCont; } @@ -231,4 +235,4 @@ SStatisInfo dnodeGetStatisInfo() { } return info; -} \ No newline at end of file +} diff --git a/src/dnode/src/dnodeStep.c b/src/dnode/src/dnodeStep.c index 2354b1d5a382d4ca7dea523cfe675883ae9d1570..eda6fb227dd6bf29499933383bb82f83ec851ee4 100644 --- a/src/dnode/src/dnodeStep.c +++ b/src/dnode/src/dnodeStep.c @@ -69,6 +69,6 @@ int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) { return 0; } -void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { - return taosStepCleanupImp(pSteps, stepSize - 1); -} \ No newline at end of file +void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) { + taosStepCleanupImp(pSteps, stepSize - 1); +} diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index a4d7e791e650352433b186bc404b2d2e11729e36..e49b3eba99408c9453189106dc6f01e0d0afc7fe 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -19,8 +19,10 @@ #include "tconfig.h" #include "dnodeMain.h" -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); static tsem_t exitSem; +static void siguser1Handler(int32_t signum, void *sigInfo, void *context); +static void siguser2Handler(int32_t signum, void *sigInfo, void *context); +static void sigintHandler(int32_t signum, void *sigInfo, void *context); int32_t main(int32_t argc, char *argv[]) { int dump_config = 0; @@ -28,7 +30,7 @@ int32_t main(int32_t argc, char *argv[]) { // Set global configuration file for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { + if (i < argc - 1) { if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { printf("config file path overflow"); exit(EXIT_FAILURE); @@ -80,8 +82,8 @@ int32_t main(int32_t argc, char *argv[]) { taosSetRandomFileFailOutput(NULL); } } else if (strcmp(argv[i], "--random-file-fail-factor") == 0) { - if ( (i+1) < argc ) { - int factor = atoi(argv[i+1]); + if ((i + 1) < argc) { + int factor = atoi(argv[i + 1]); printf("The factor of random failure is %d\n", factor); taosSetRandomFileFailFactor(factor); } else { @@ -93,7 +95,7 @@ int32_t main(int32_t argc, char *argv[]) { } if (0 != dump_config) { - tscEmbedded = 1; + tscEmbedded = 1; taosInitGlobalCfg(); taosReadGlobalLogCfg(); @@ -112,14 +114,13 @@ int32_t main(int32_t argc, char *argv[]) { } /* Set termination handler. */ - struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = signal_handler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); + taosSetSignal(SIGUSR1, siguser1Handler); + taosSetSignal(SIGUSR2, siguser2Handler); + taosSetSignal(SIGTERM, sigintHandler); + taosSetSignal(SIGHUP, sigintHandler); + taosSetSignal(SIGINT, sigintHandler); + taosSetSignal(SIGABRT, sigintHandler); + taosSetSignal(SIGBREAK, sigintHandler); // Open /var/log/syslog file to record information. openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1); @@ -144,33 +145,40 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Shut down TDengine service successfully"); dInfo("TDengine is shut down!"); closelog(); + +#ifdef WINDOWS + tsem_post(&exitSem); +#endif return EXIT_SUCCESS; } -static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { - if (signum == SIGUSR1) { - taosCfgDynamicOptions("debugFlag 143"); - return; - } - if (signum == SIGUSR2) { - taosCfgDynamicOptions("resetlog"); - return; - } +static void siguser1Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("debugFlag 143"); } - syslog(LOG_INFO, "Shut down signal is %d", signum); - syslog(LOG_INFO, "Shutting down TDengine service..."); - // clean the system. - dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid)); +static void siguser2Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("resetlog"); } +static void sigintHandler(int32_t signum, void *sigInfo, void *context) { // protect the application from receive another signal - struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); - sigaction(SIGUSR1, &act, NULL); - sigaction(SIGUSR2, &act, NULL); + taosIgnSignal(SIGUSR1); + taosIgnSignal(SIGUSR2); + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGBREAK); + + // clean the system. + dInfo("shut down signal is %d", signum); + +#ifndef WINDOWS + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); +#endif + + syslog(LOG_INFO, "Shut down signal is %d", signum); + syslog(LOG_INFO, "Shutting down TDengine service..."); // inform main thread to exit tsem_post(&exitSem); +#ifdef WINDOWS + tsem_wait(&exitSem); +#endif } diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index ff9598ecc563ac48bac051027385ab188e94ce75..4caece16612353155886ff20055433878ec7411c 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -31,12 +31,16 @@ #include "mnodeAcct.h" #include "dnodeTelemetry.h" -static tsem_t tsExitSem; +// sem_timedwait is NOT implemented on MacOSX +// thus, we use pthread_mutex_t/pthread_cond_t to simulate +static pthread_mutex_t tsExitLock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t tsExitCond = PTHREAD_COND_INITIALIZER; +static volatile int tsExit = 0; static pthread_t tsTelemetryThread; #define TELEMETRY_SERVER "telemetry.taosdata.com" #define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 +#define REPORT_INTERVAL 86400 static void beginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); @@ -93,14 +97,14 @@ static void addStringField(SBufferWriter* bw, const char* k, const char* v) { static void addCpuInfo(SBufferWriter* bw) { char * line = NULL; size_t size = 0; - int done = 0; + int32_t done = 0; FILE* fp = fopen("/proc/cpuinfo", "r"); if (fp == NULL) { return; } - while (done != 3 && (size = getline(&line, &size, fp)) != -1) { + while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (((done&1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -129,7 +133,7 @@ static void addOsInfo(SBufferWriter* bw) { return; } - while ((size = getline(&line, &size, fp)) != -1) { + while ((size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -155,7 +159,7 @@ static void addMemoryInfo(SBufferWriter* bw) { return; } - while ((size = getline(&line, &size, fp)) != -1) { + while ((size = tgetline(&line, &size, fp)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -172,8 +176,8 @@ static void addMemoryInfo(SBufferWriter* bw) { static void addVersionInfo(SBufferWriter* bw) { addStringField(bw, "version", version); addStringField(bw, "buildInfo", buildinfo); - addStringField(bw, "gitInfo", gitinfo); - addStringField(bw, "email", tsEmail); + addStringField(bw, "gitInfo", gitinfo); + addStringField(bw, "email", tsEmail); } static void addRuntimeInfo(SBufferWriter* bw) { @@ -200,7 +204,7 @@ static void sendTelemetryReport() { dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno)); return; } - int fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); if (fd < 0) { dTrace("failed to create socket for telemetry, reason:%s", strerror(errno)); return; @@ -222,10 +226,10 @@ static void sendTelemetryReport() { "Content-Type: application/json\n" "Content-Length: "; - taosWriteSocket(fd, header, strlen(header)); - int contLen = tbufTell(&bw) - 1; + taosWriteSocket(fd, header, (int32_t)strlen(header)); + int32_t contLen = (int32_t)(tbufTell(&bw) - 1); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, strlen(buf)); + taosWriteSocket(fd, buf, (int32_t)strlen(buf)); taosWriteSocket(fd, tbufGetData(&bw, false), contLen); tbufCloseWriter(&bw); @@ -241,12 +245,14 @@ static void* telemetryThread(void* param) { clock_gettime(CLOCK_REALTIME, &end); end.tv_sec += 300; // wait 5 minutes before send first report - while (1) { - if (sem_timedwait(&tsExitSem, &end) == 0) { - break; - } else if (errno != ETIMEDOUT) { - continue; - } + while (!tsExit) { + int r = 0; + struct timespec ts = end; + pthread_mutex_lock(&tsExitLock); + r = pthread_cond_timedwait(&tsExitCond, &tsExitLock, &ts); + pthread_mutex_unlock(&tsExitLock); + if (r==0) break; + if (r!=ETIMEDOUT) continue; if (sdbIsMaster()) { sendTelemetryReport(); @@ -258,31 +264,24 @@ static void* telemetryThread(void* param) { } static void dnodeGetEmail(char* filepath) { - int fd = open(filepath, O_RDONLY); + int32_t fd = open(filepath, O_RDONLY); if (fd < 0) { return; } - + if (taosRead(fd, (void *)tsEmail, TSDB_FQDN_LEN) < 0) { dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); - } + } - close(fd); + taosClose(fd); } - int32_t dnodeInitTelemetry() { if (!tsEnableTelemetryReporting) { return 0; } - dnodeGetEmail("/usr/local/taos/email"); - - if (tsem_init(&tsExitSem, 0, 0) == -1) { - // just log the error, it is ok for telemetry to fail - dTrace("failed to create semaphore for telemetry, reason:%s", strerror(errno)); - return 0; - } + dnodeGetEmail("/usr/local/taos/email"); pthread_attr_t attr; pthread_attr_init(&attr); @@ -291,7 +290,7 @@ int32_t dnodeInitTelemetry() { int32_t code = pthread_create(&tsTelemetryThread, &attr, telemetryThread, NULL); pthread_attr_destroy(&attr); if (code != 0) { - dTrace("failed to create telemetry thread, reason:%s", strerror(errno)); + dTrace("failed to create telemetry thread, reason:%s", strerror(code)); } dInfo("dnode telemetry is initialized"); @@ -303,9 +302,12 @@ void dnodeCleanupTelemetry() { return; } - if (tsTelemetryThread) { - tsem_post(&tsExitSem); + if (taosCheckPthreadValid(tsTelemetryThread)) { + pthread_mutex_lock(&tsExitLock); + tsExit = 1; + pthread_cond_signal(&tsExitCond); + pthread_mutex_unlock(&tsExitLock); + pthread_join(tsTelemetryThread, NULL); - tsem_destroy(&tsExitSem); } } diff --git a/src/dnode/src/dnodeVMgmt.c b/src/dnode/src/dnodeVMgmt.c index 4a3d6d9a84f7918c8cbbc40cd80b074ff164cf85..66c94bf6755850f4b731e425cd0da90f75c218f3 100644 --- a/src/dnode/src/dnodeVMgmt.c +++ b/src/dnode/src/dnodeVMgmt.c @@ -30,6 +30,7 @@ static taos_queue tsVMgmtQueue = NULL; static void * dnodeProcessMgmtQueue(void *param); static int32_t dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg); +static int32_t dnodeProcessSyncVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *pMsg); static int32_t dnodeProcessAlterStreamMsg(SRpcMsg *pMsg); static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg); @@ -39,6 +40,7 @@ static int32_t (*dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *pMsg); int32_t dnodeInitVMgmt() { dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CREATE_VNODE] = dnodeProcessCreateVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_VNODE] = dnodeProcessAlterVnodeMsg; + dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_SYNC_VNODE] = dnodeProcessSyncVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_DROP_VNODE] = dnodeProcessDropVnodeMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_ALTER_STREAM] = dnodeProcessAlterStreamMsg; dnodeProcessMgmtMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeProcessConfigDnodeMsg; @@ -174,11 +176,18 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) { vnodeRelease(pVnode); return code; } else { - dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId); + dInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } } +static int32_t dnodeProcessSyncVnodeMsg(SRpcMsg *rpcMsg) { + SSyncVnodeMsg *pSyncVnode = rpcMsg->pCont; + pSyncVnode->vgId = htonl(pSyncVnode->vgId); + + return vnodeSync(pSyncVnode->vgId); +} + static int32_t dnodeProcessDropVnodeMsg(SRpcMsg *rpcMsg) { SDropVnodeMsg *pDrop = rpcMsg->pCont; pDrop->vgId = htonl(pDrop->vgId); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index ab5a4ccaad5fa3e5969776c75db26a30d3e32e8e..84fd260d91420ea19a796f10fa96d1fbbb1ac648 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -52,14 +52,14 @@ int32_t dnodeInitVWrite() { void dnodeCleanupVWrite() { for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (pWorker->thread) { - taosQsetThreadResume(pWorker->qset); + if (taosCheckPthreadValid(pWorker->thread)) { + if (pWorker->qset) taosQsetThreadResume(pWorker->qset); } } for (int32_t i = 0; i < tsVWriteWP.max; ++i) { SVWriteWorker *pWorker = tsVWriteWP.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); taosFreeQall(pWorker->qall); taosCloseQset(pWorker->qset); @@ -205,7 +205,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead.msgType], qtypeStr[qtype], pWrite->pHead.version); pWrite->code = vnodeProcessWrite(pVnode, &pWrite->pHead, qtype, pWrite); - if (pWrite->code <= 0) pWrite->processedCount = 1; + if (pWrite->code <= 0) atomic_add_fetch_32(&pWrite->processedCount, 1); if (pWrite->code > 0) pWrite->code = 0; if (pWrite->code == 0 && pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true; @@ -222,7 +222,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code); } else { if (qtype == TAOS_QTYPE_FWD) { - vnodeConfirmForward(pVnode, pWrite->pHead.version, 0); + vnodeConfirmForward(pVnode, pWrite->pHead.version, 0, pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT); } if (pWrite->rspRet.rsp) { rpcFreeCont(pWrite->rspRet.rsp); diff --git a/src/dnode/src/dnodeVnodes.c b/src/dnode/src/dnodeVnodes.c index c62d5a8207c8cd5f540e3ff861d44cd55c7c8220..d96251cebe1cbbcddeb48f1a09091e1c09caf630 100644 --- a/src/dnode/src/dnodeVnodes.c +++ b/src/dnode/src/dnodeVnodes.c @@ -157,7 +157,7 @@ int32_t dnodeInitVnodes() { int32_t failedVnodes = 0; for (int32_t t = 0; t < threadNum; ++t) { SOpenVnodeThread *pThread = &threads[t]; - if (pThread->vnodeNum > 0 && pThread->thread) { + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { pthread_join(pThread->thread, NULL); } openVnodes += pThread->opened; @@ -198,6 +198,15 @@ void dnodeCleanupVnodes() { static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { if (pMsg->code != TSDB_CODE_SUCCESS) { dError("status rsp is received, error:%s", tstrerror(pMsg->code)); + if (pMsg->code == TSDB_CODE_MND_DNODE_NOT_EXIST) { + char clusterId[TSDB_CLUSTER_ID_LEN]; + dnodeGetClusterId(clusterId); + if (clusterId[0] != '\0') { + dError("exit zombie dropped dnode"); + exit(EXIT_FAILURE); + } + } + taosTmrReset(dnodeSendStatusMsg, tsStatusInterval * 1000, NULL, tsDnodeTmr, &tsStatusTimer); return; } @@ -260,7 +269,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); pStatus->clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java b/src/inc/module.h similarity index 70% rename from src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java rename to src/inc/module.h index c1b9b02fa85f8e2197f41d75f8f0c25b3f4c416c..b9b64c493ec5777ddd6981ebf3212ffd0ff0114d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java +++ b/src/inc/module.h @@ -1,4 +1,4 @@ -/*************************************************************************** +/* * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify @@ -11,9 +11,20 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - *****************************************************************************/ -package com.taosdata.jdbc; + */ -public interface TSDBSubscribeCallBack { - void invoke(TSDBResultSet resultSet); +#ifndef TDENGINE_MODULE +#define TDENGINE_MODULE + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t moduleStart(); +void moduleStop(); + +#ifdef __cplusplus } +#endif + +#endif diff --git a/src/inc/query.h b/src/inc/query.h index 7342221cb9de1b632ad0f398f2f3a8d27621747a..c9dabcef5454d8322ab64eade53802a3e29a7b53 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -28,7 +28,7 @@ typedef void* qinfo_t; * @param qinfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo); +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t *qId); /** @@ -38,7 +38,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs * @param qinfo * @return */ -bool qTableQuery(qinfo_t qinfo); +bool qTableQuery(qinfo_t qinfo, uint64_t *qId); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -88,9 +88,10 @@ void* qOpenQueryMgmt(int32_t vgId); void qQueryMgmtNotifyClosed(void* pExecutor); void qQueryMgmtReOpen(void *pExecutor); void qCleanupQueryMgmt(void* pExecutor); -void** qRegisterQInfo(void* pMgmt, uint64_t qInfo); +void** qRegisterQInfo(void* pMgmt, uint64_t qId, uint64_t qInfo); void** qAcquireQInfo(void* pMgmt, uint64_t key); void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle); +bool checkQIdEqual(void *qHandle, uint64_t qId); #ifdef __cplusplus } diff --git a/src/inc/taos.h b/src/inc/taos.h index 5e4f50e31d6016ebd1e01aecb59d8646180ce76a..cd8e116053bd9adabda9a1eeeb20c6d92679d99d 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -68,7 +68,7 @@ typedef struct taosField { #define DLL_EXPORT #endif -DLL_EXPORT void taos_init(); +DLL_EXPORT int taos_init(); DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); @@ -140,7 +140,7 @@ DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param); -DLL_EXPORT void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param); +//DLL_EXPORT void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param); typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7ccb7db935e37bb2472384f031ad4b9d144135fd..024bc198df80bd42f547cd89543c028403acccd3 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -36,29 +36,6 @@ extern "C" { #define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX}) #define TSKEY_INITIAL_VAL INT64_MIN -// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR -typedef int32_t VarDataOffsetT; -typedef int16_t VarDataLenT; - -typedef struct tstr { - VarDataLenT len; - char data[]; -} tstr; - -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) - -#define varDataLen(v) ((VarDataLenT *)(v))[0] -#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) -#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) -#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) -#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) - -// this data type is internally used only in 'in' query to hold the values -#define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) - - // Bytes for each type. extern const int32_t TYPE_BYTES[15]; @@ -164,70 +141,6 @@ do { \ #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } #endif -typedef struct tDataTypeDescriptor { - int16_t nType; - int16_t nameLen; - int32_t nSize; - char * aName; - int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize); - int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize); - void (*getStatisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minindex, int16_t *maxindex, int16_t *numofnull); -} tDataTypeDescriptor; - -extern tDataTypeDescriptor tDataTypeDesc[15]; - -bool isValidDataType(int32_t type); - -static FORCE_INLINE bool isNull(const char *val, int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; - case TSDB_DATA_TYPE_TINYINT: - return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; - case TSDB_DATA_TYPE_SMALLINT: - return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; - case TSDB_DATA_TYPE_INT: - return *(uint32_t *)val == TSDB_DATA_INT_NULL; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; - case TSDB_DATA_TYPE_FLOAT: - return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; - case TSDB_DATA_TYPE_DOUBLE: - return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; - case TSDB_DATA_TYPE_NCHAR: - return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; - case TSDB_DATA_TYPE_BINARY: - return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; - case TSDB_DATA_TYPE_UTINYINT: - return *(uint8_t*) val == TSDB_DATA_UTINYINT_NULL; - case TSDB_DATA_TYPE_USMALLINT: - return *(uint16_t*) val == TSDB_DATA_USMALLINT_NULL; - case TSDB_DATA_TYPE_UINT: - return *(uint32_t*) val == TSDB_DATA_UINT_NULL; - case TSDB_DATA_TYPE_UBIGINT: - return *(uint64_t*) val == TSDB_DATA_UBIGINT_NULL; - - default: - return false; - }; -} - -void setVardataNull(char* val, int32_t type); -void setNull(char *val, int32_t type, int32_t bytes); -void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); -void* getNullValue(int32_t type); - -void assignVal(char *val, const char *src, int32_t len, int32_t type); -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); - -int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned); - -#define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) - // TODO: check if below is necessary #define TSDB_RELATION_INVALID 0 #define TSDB_RELATION_LESS 1 @@ -250,6 +163,11 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo #define TSDB_BINARY_OP_MULTIPLY 32 #define TSDB_BINARY_OP_DIVIDE 33 #define TSDB_BINARY_OP_REMAINDER 34 + + +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) <= TSDB_RELATION_IN)) +#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) + #define TS_PATH_DELIMITER_LEN 1 #define TSDB_UNI_LEN 24 @@ -270,7 +188,7 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_SQL_SHOW_LEN 512 -#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024U) // sql length should be less than 1mb +#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb #define TSDB_APPNAME_LEN TSDB_UNI_LEN @@ -304,6 +222,9 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo #define TSDB_MQTT_TOPIC_LEN 64 #define TSDB_MQTT_CLIENT_ID_LEN 32 +#define TSDB_DB_TYPE_DEFAULT 0 +#define TSDB_DB_TYPE_TOPIC 1 + #define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE @@ -322,7 +243,9 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo #define TSDB_MAX_REPLICA 5 #define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_BLOCK_DIST_COLUMN_INDEX (-2) #define TSDB_UD_COLUMN_INDEX (-100) + #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 @@ -366,7 +289,7 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo #define TSDB_MAX_COMP_LEVEL 2 #define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_WAL_LEVEL 1 +#define TSDB_MIN_WAL_LEVEL 0 #define TSDB_MAX_WAL_LEVEL 2 #define TSDB_DEFAULT_WAL_LEVEL 1 @@ -386,11 +309,15 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo #define TSDB_MAX_DB_REPLICA_OPTION 3 #define TSDB_DEFAULT_DB_REPLICA_OPTION 1 +#define TSDB_MIN_DB_PARTITON_OPTION 0 +#define TSDB_MAX_DB_PARTITON_OPTION 1000 +#define TSDB_DEFAULT_DB_PARTITON_OPTION 4 + #define TSDB_MIN_DB_QUORUM_OPTION 1 #define TSDB_MAX_DB_QUORUM_OPTION 2 #define TSDB_DEFAULT_DB_QUORUM_OPTION 1 -#define TSDB_MAX_JOIN_TABLE_NUM 5 +#define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 #define TSDB_MAX_BINARY_LEN (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE) @@ -399,8 +326,8 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo #define TSDB_MAX_RPC_THREADS 5 -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode /* * 1. ordinary sub query for select * from super_table @@ -420,29 +347,29 @@ int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bo #define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u #define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type -#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) -#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) -#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) -#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) - -#define TSDB_ORDER_ASC 1 -#define TSDB_ORDER_DESC 2 - -#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 -#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 -#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 -#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 -#define TSDB_DEFAULT_USERS_HASH_SIZE 20 -#define TSDB_DEFAULT_DBS_HASH_SIZE 100 -#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 -#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 -#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 - -#define TSDB_PORT_DNODESHELL 0 -#define TSDB_PORT_DNODEDNODE 5 -#define TSDB_PORT_SYNC 10 -#define TSDB_PORT_HTTP 11 -#define TSDB_PORT_ARBITRATOR 12 +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) + +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 + +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 + +#define TSDB_PORT_DNODESHELL 0 +#define TSDB_PORT_DNODEDNODE 5 +#define TSDB_PORT_SYNC 10 +#define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 #define TSDB_MAX_WAL_SIZE (1024*1024*3) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index b6cf1b5f1f56cc4eaae652907fc14b6cf8965b9e..8daccee1f255e5956bb8308bac5d8d05d1088642 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -23,11 +23,7 @@ extern "C" { #include #include -#ifdef TAOS_ERROR_C -#define TAOS_DEFINE_ERROR(name, mod, code, msg) {.val = (int32_t)((0x80000000 | ((mod)<<16) | (code))), .str=(msg)}, -#else -#define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = (int32_t)((0x80000000 | ((mod)<<16) | (code))); -#endif +#define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) #define TAOS_SUCCEEDED(err) ((err) >= 0) @@ -40,395 +36,394 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SUCCESS 0 -#ifdef TAOS_ERROR_C -static STaosError errors[] = { - {.val = 0, .str = "success"}, -#endif - // rpc -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ACTION_IN_PROGRESS, 0, 0x0001, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_REQUIRED, 0, 0x0002, "Authentication required") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, 0, 0x0003, "Authentication failure") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, 0, 0x0004, "Redirect") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NOT_READY, 0, 0x0005, "System not ready") // peer is not ready to process data -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ALREADY_PROCESSED, 0, 0x0006, "Message already processed") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED,0, 0x0007, "Last session not finished") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MISMATCHED_LINK_ID, 0, 0x0008, "Mismatched meter id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TOO_SLOW, 0, 0x0009, "Processing of request timed out") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, 0, 0x000A, "Number of sessions reached limit") // too many sessions -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, 0, 0x000B, "Unable to establish connection") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_APP_ERROR, 0, 0x000C, "Unexpected generic error in RPC") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_UNEXPECTED_RESPONSE, 0, 0x000D, "Unexpected response") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VALUE, 0, 0x000E, "Invalid value") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID, 0, 0x000F, "Invalid transaction id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, 0, 0x0010, "Invalid session id") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid message type") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Client and server's time is not synchronized") -TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VERSION, 0, 0x0016, "Invalid app version") +#define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //"Action in progress") +#define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //"Authentication required") +#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //"Authentication failure") +#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) //"Redirect") +#define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //"System not ready") // peer is not ready to process data +#define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //"Message already processed") +#define TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x0007) //"Last session not finished") +#define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //"Mismatched meter id") +#define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //"Processing of request timed out") +#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //"Number of sessions reached limit") // too many sessions +#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) //"Unable to establish connection") +#define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //"Unexpected generic error in RPC") +#define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //"Unexpected response") +#define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //"Invalid value") +#define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //"Invalid transaction id") +#define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //"Invalid session id") +#define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //"Invalid message type") +#define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //"Invalid response type") +#define TSDB_CODE_RPC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0013) //"Client and server's time is not synchronized") +#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) //"Database not ready") +#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) //"Unable to resolve FQDN") +#define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //"Invalid app version") //common & util -TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "Operation not supported") -TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, 0, 0x0101, "Memory corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, 0, 0x0102, "Out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, 0, 0x0103, "Invalid config message") -TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, 0, 0x0104, "Data file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, 0, 0x0105, "Ref out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, 0, 0x0106, "too many Ref Objs") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, 0, 0x0107, "Ref ID is removed") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, 0, 0x0108, "Invalid Ref ID") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, 0, 0x0109, "Ref is already there") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, 0, 0x010A, "Ref is not there") +#define TSDB_CODE_COM_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) //"Operation not supported") +#define TSDB_CODE_COM_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) //"Memory corrupted") +#define TSDB_CODE_COM_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0102) //"Out of memory") +#define TSDB_CODE_COM_INVALID_CFG_MSG TAOS_DEF_ERROR_CODE(0, 0x0103) //"Invalid config message") +#define TSDB_CODE_COM_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) //"Data file corrupted") +#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) //"Ref out of memory") +#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) //"too many Ref Objs") +#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) //"Ref ID is removed") +#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) //"Invalid Ref ID") +#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) //"Ref is already there") +#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) //"Ref is not there") //client -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, 0, 0x0200, "Invalid SQL statement") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE, 0, 0x0201, "Invalid qhandle") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TIME_STAMP, 0, 0x0202, "Invalid combination of client/service time") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VALUE, 0, 0x0203, "Invalid value in client") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VERSION, 0, 0x0204, "Invalid client version") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_IE, 0, 0x0205, "Invalid client ie") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_FQDN, 0, 0x0206, "Invalid host name") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_USER_LENGTH, 0, 0x0207, "Invalid user name") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH, 0, 0x0208, "Invalid password") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH, 0, 0x0209, "Database name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH, 0, 0x020A, "Table name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION, 0, 0x020B, "Invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY, 0, 0x020C, "System out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_DISKSPACE, 0, 0x020D, "System out of disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED, 0, 0x020E, "Query cache erased") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED, 0, 0x020F, "Query terminated") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY, 0, 0x0210, "Result set too large to be sorted") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Application error") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax error in SQL") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED, 0, 0x0217, "Database not specified or available") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, 0, 0x0218, "Table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, 0, 0x0219, "SQL statement too long, check maxSQLLength config") +#define TSDB_CODE_TSC_INVALID_SQL TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid SQL statement") +#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) //"Invalid qhandle") +#define TSDB_CODE_TSC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0202) //"Invalid combination of client/service time") +#define TSDB_CODE_TSC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x0203) //"Invalid value in client") +#define TSDB_CODE_TSC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0204) //"Invalid client version") +#define TSDB_CODE_TSC_INVALID_IE TAOS_DEF_ERROR_CODE(0, 0x0205) //"Invalid client ie") +#define TSDB_CODE_TSC_INVALID_FQDN TAOS_DEF_ERROR_CODE(0, 0x0206) //"Invalid host name") +#define TSDB_CODE_TSC_INVALID_USER_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0207) //"Invalid user name") +#define TSDB_CODE_TSC_INVALID_PASS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0208) //"Invalid password") +#define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) //"Database name too long") +#define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) //"Table name too long") +#define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) //"Invalid connection") +#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) //"System out of memory") +#define TSDB_CODE_TSC_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x020D) //"System out of disk space") +#define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) //"Query cache erased") +#define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) //"Query terminated") +#define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) //"Result set too large to be sorted") // too many result for ordered super table projection query +#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) //"Application error") +#define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) //"Action in progress") +#define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) //"Disconnected from service") +#define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) //"No write permission") +#define TSDB_CODE_TSC_CONN_KILLED TAOS_DEF_ERROR_CODE(0, 0x0215) //"Connection killed") +#define TSDB_CODE_TSC_SQL_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x0216) //"Syntax error in SQL") +#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217) //"Database not specified or available") +#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218) //"Table does not exist") +#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219) //"SQL statement too long check maxSQLLength config") // mnode -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, 0, 0x0301, "Message is progressing") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, 0, 0x0302, "Messag need to be reprocessed") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, 0, 0x0303, "Insufficient privilege for operation") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, 0, 0x0304, "Unexpected generic error in mnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, 0, 0x0305, "Invalid message connection") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION, 0, 0x0306, "Incompatible protocol version") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN, 0, 0x0307, "Invalid message length") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE, 0, 0x0308, "Invalid message type") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS, 0, 0x0309, "Too many connections") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_OUT_OF_MEMORY, 0, 0x030A, "Out of memory in mnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, 0, 0x030B, "Data expired") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, 0, 0x030C, "Invalid query id") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, 0, 0x030D, "Invalid stream id") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, 0, 0x030E, "Invalid connection id") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_IS_RUNNING, 0, 0x0310, "mnode is alreay running") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC, 0, 0x0311, "failed to config sync") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_START_SYNC, 0, 0x0312, "failed to start sync") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CREATE_DIR, 0, 0x0313, "failed to create mnode dir") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_INIT_STEP, 0, 0x0314, "failed to init components") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, 0, 0x0320, "Object already there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, 0, 0x0321, "Unexpected generic error in sdb") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE, 0, 0x0322, "Invalid table type") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE, 0, 0x0323, "Object not there") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW, 0, 0x0324, "Invalid meta row") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE, 0, 0x0325, "Invalid key type") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, 0, 0x0330, "DNode already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, 0, 0x0331, "DNode does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, 0, 0x0332, "VGroup does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, 0, 0x0333, "Master DNode cannot be removed") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, 0, 0x0334, "Out of DNodes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "Cluster cfg inconsistent") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, 0, 0x0336, "Invalid dnode cfg option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED, 0, 0x0337, "Balance already enabled") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, 0, 0x0338, "Vgroup not in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, 0, 0x0339, "Vgroup already in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE, 0, 0x033A, "Dnode not avaliable") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, 0, 0x033B, "Cluster id not match") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, 0, 0x033C, "Cluster not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED, 0, 0x033D, "Dnode Id not configured") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED, 0, 0x033E, "Dnode Ep not configured") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "Account already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "Invalid account") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, 0, 0x0342, "Invalid account options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, 0, 0x0343, "Account authorization has expired") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, 0, 0x0350, "User already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, 0, 0x0351, "Invalid user") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT, 0, 0x0352, "Invalid user format") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT, 0, 0x0353, "Invalid password format") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, 0, 0x0354, "Can not get user from conn") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, 0, 0x0355, "Too many users") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST, 0, 0x0360, "Table already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, 0, 0x0361, "Table name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, 0, 0x0362, "Table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, 0, 0x0363, "Invalid table type in tsdb") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, 0, 0x0364, "Too many tags") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, 0, 0x0366, "Too many time series") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, 0, 0x0367, "Not super table") // operation only available for super table -TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, 0, 0x0368, "Tag name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, 0, 0x0369, "Tag already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, 0, 0x036A, "Tag does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, 0, 0x036B, "Field already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, 0, 0x036C, "Field does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, 0, 0x036D, "Super table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, 0, 0x036E, "Invalid create table message") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, 0, 0x0380, "Database not specified or available") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, 0, 0x0381, "Database already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, 0, 0x0382, "Invalid database options") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "Invalid database name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "Cannot delete monitor database") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many databases for account") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced") - -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS, 0, 0x0390, "Invalid database option: days out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP, 0, 0x0391, "Invalid database option: keep >= keep2 >= keep1 >= days") +#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed") +#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) //"Message is progressing") +#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302) //"Messag need to be reprocessed") +#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) //"Insufficient privilege for operation") +#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0304) //"Unexpected generic error in mnode") +#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) //"Invalid message connection") +#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0306) //"Incompatible protocol version") +#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0307) //"Invalid message length") +#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0308) //"Invalid message type") +#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x0309) //"Too many connections") +#define TSDB_CODE_MND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x030A) //"Out of memory in mnode") +#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x030B) //"Data expired") +#define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x030C) //"Invalid query id") +#define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D) //"Invalid stream id") +#define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x030E) //"Invalid connection id") +#define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0310) //"mnode is alreay running") +#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0311) //"failed to config sync") +#define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0312) //"failed to start sync") +#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0313) //"failed to create mnode dir") +#define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) //"failed to init components") + +#define TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) //"Object already there") +#define TSDB_CODE_MND_SDB_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) //"Unexpected generic error in sdb") +#define TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0322) //"Invalid table type") +#define TSDB_CODE_MND_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0323) //"Object not there") +#define TSDB_CODE_MND_SDB_INVAID_META_ROW TAOS_DEF_ERROR_CODE(0, 0x0324) //"Invalid meta row") +#define TSDB_CODE_MND_SDB_INVAID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) //"Invalid key type") + +#define TSDB_CODE_MND_DNODE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0330) //"DNode already exists") +#define TSDB_CODE_MND_DNODE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0331) //"DNode does not exist") +#define TSDB_CODE_MND_VGROUP_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0332) //"VGroup does not exist") +#define TSDB_CODE_MND_NO_REMOVE_MASTER TAOS_DEF_ERROR_CODE(0, 0x0333) //"Master DNode cannot be removed") +#define TSDB_CODE_MND_NO_ENOUGH_DNODES TAOS_DEF_ERROR_CODE(0, 0x0334) //"Out of DNodes") +#define TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT TAOS_DEF_ERROR_CODE(0, 0x0335) //"Cluster cfg inconsistent") +#define TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION TAOS_DEF_ERROR_CODE(0, 0x0336) //"Invalid dnode cfg option") +#define TSDB_CODE_MND_BALANCE_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0337) //"Balance already enabled") +#define TSDB_CODE_MND_VGROUP_NOT_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0338) //"Vgroup not in dnode") +#define TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE TAOS_DEF_ERROR_CODE(0, 0x0339) //"Vgroup already in dnode") +#define TSDB_CODE_MND_DNODE_NOT_FREE TAOS_DEF_ERROR_CODE(0, 0x033A) //"Dnode not avaliable") +#define TSDB_CODE_MND_INVALID_CLUSTER_ID TAOS_DEF_ERROR_CODE(0, 0x033B) //"Cluster id not match") +#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x033C) //"Cluster not ready") +#define TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED TAOS_DEF_ERROR_CODE(0, 0x033D) //"Dnode Id not configured") +#define TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED TAOS_DEF_ERROR_CODE(0, 0x033E) //"Dnode Ep not configured") + +#define TSDB_CODE_MND_ACCT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0340) //"Account already exists") +#define TSDB_CODE_MND_INVALID_ACCT TAOS_DEF_ERROR_CODE(0, 0x0341) //"Invalid account") +#define TSDB_CODE_MND_INVALID_ACCT_OPTION TAOS_DEF_ERROR_CODE(0, 0x0342) //"Invalid account options") +#define TSDB_CODE_MND_ACCT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0343) //"Account authorization has expired") + +#define TSDB_CODE_MND_USER_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0350) //"User already exists") +#define TSDB_CODE_MND_INVALID_USER TAOS_DEF_ERROR_CODE(0, 0x0351) //"Invalid user") +#define TSDB_CODE_MND_INVALID_USER_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0352) //"Invalid user format") +#define TSDB_CODE_MND_INVALID_PASS_FORMAT TAOS_DEF_ERROR_CODE(0, 0x0353) //"Invalid password format") +#define TSDB_CODE_MND_NO_USER_FROM_CONN TAOS_DEF_ERROR_CODE(0, 0x0354) //"Can not get user from conn") +#define TSDB_CODE_MND_TOO_MANY_USERS TAOS_DEF_ERROR_CODE(0, 0x0355) //"Too many users") + +#define TSDB_CODE_MND_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) //"Table already exists") +#define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) //"Table name too long") +#define TSDB_CODE_MND_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0362) //"Table does not exist") +#define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) //"Invalid table type in tsdb") +#define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x0364) //"Too many tags") +#define TSDB_CODE_MND_TOO_MANY_TIMESERIES TAOS_DEF_ERROR_CODE(0, 0x0366) //"Too many time series") +#define TSDB_CODE_MND_NOT_SUPER_TABLE TAOS_DEF_ERROR_CODE(0, 0x0367) //"Not super table") // operation only available for super table +#define TSDB_CODE_MND_COL_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x0368) //"Tag name too long") +#define TSDB_CODE_MND_TAG_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0369) //"Tag already exists") +#define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036A) //"Tag does not exist") +#define TSDB_CODE_MND_FIELD_ALREAY_EXIST TAOS_DEF_ERROR_CODE(0, 0x036B) //"Field already exists") +#define TSDB_CODE_MND_FIELD_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) //"Field does not exist") +#define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) //"Super table does not exist") +#define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message") + +#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available") +#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists") +#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options") +#define TSDB_CODE_MND_INVALID_DB TAOS_DEF_ERROR_CODE(0, 0x0383) //"Invalid database name") +#define TSDB_CODE_MND_MONITOR_DB_FORBIDDEN TAOS_DEF_ERROR_CODE(0, 0x0384) //"Cannot delete monitor database") +#define TSDB_CODE_MND_TOO_MANY_DATABASES TAOS_DEF_ERROR_CODE(0, 0x0385) //"Too many databases for account") +#define TSDB_CODE_MND_DB_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0386) //"Database not available") +#define TSDB_CODE_MND_VGROUP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0387) //"Database unsynced") + +#define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) //"Invalid database option: days out of range") +#define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) //"Invalid database option: keep >= keep1 >= keep0 >= days") + +#define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x0392) //"Invalid topic name) +#define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x0393) //"Invalid topic option) +#define TSDB_CODE_MND_INVALID_TOPIC_PARTITONS TAOS_DEF_ERROR_CODE(0, 0x0394) //"Invalid topic partitons num, valid range: [1, 1000]) +#define TSDB_CODE_MND_TOPIC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0395) //"Topic already exists) // dnode -TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, 0, 0x0402, "No permission for disk files in dnode") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, 0, 0x0403, "Invalid message length") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, 0, 0x0404, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES, 0, 0x0405, "Too many vnode directories") - -// vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, 0, 0x0500, "Action in progress") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, 0, 0x0501, "Message not processed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, 0, 0x0502, "Action need to be reprocessed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, 0, 0x0503, "Invalid Vgroup ID") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED, 0, 0x0504, "Vnode initialization failed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISKSPACE, 0, 0x0505, "System out of disk space") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS, 0, 0x0506, "No write permission for disk files") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, 0, 0x0507, "Missing data file") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "Out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected generic error in vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid version file") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Database memory is full for commit failed") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, 0, 0x050C, "Database memory is full for waiting commit") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_DROPPING, 0, 0x050D, "Database is dropping") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_BALANCING, 0, 0x050E, "Database is balancing") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Database write operation denied") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, 0, 0x0513, "Database is syncing") +#define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) //"Message not processed") +#define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) //"Dnode out of memory") +#define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0402) //"No permission for disk files in dnode") +#define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0403) //"Invalid message length") +#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0404) //"Action in progress") +#define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) //"Too many vnode directories") + +// vnode +#define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") +#define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) //"Message not processed") +#define TSDB_CODE_VND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0502) //"Action need to be reprocessed") +#define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) //"Invalid Vgroup ID") +#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) //"Vnode initialization failed") +#define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) //"System out of disk space") +#define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) //"No write permission for disk files") +#define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) //"Missing data file") +#define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) //"Out of memory") +#define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) //"Unexpected generic error in vnode") +#define TSDB_CODE_VND_INVALID_VRESION_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) //"Invalid version file") +#define TSDB_CODE_VND_IS_FULL TAOS_DEF_ERROR_CODE(0, 0x050B) //"Database memory is full for commit failed") +#define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) //"Database memory is full for waiting commit") +#define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) //"Database is dropping") +#define TSDB_CODE_VND_IS_BALANCING TAOS_DEF_ERROR_CODE(0, 0x050E) //"Database is balancing") +#define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) //"Database suspended") +#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) //"Database write operation denied") +#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) //"Database is syncing") // tsdb -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "Invalid table ID") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, 0, 0x0601, "Invalid table type") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, 0, 0x0602, "Invalid table schema version") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, 0, 0x0603, "Table already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, 0, 0x0604, "Invalid configuration") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, 0, 0x0605, "Tsdb init failed") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE, 0, 0x0606, "No diskspace for tsdb") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS, 0, 0x0607, "No permission for disk files") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, 0, 0x0608, "Data file(s) corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, 0, 0x0609, "Out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, 0, 0x060A, "Tag too old") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, 0, 0x060B, "Timestamp data out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, 0, 0x060C, "Submit message is messed up") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, 0, 0x060D, "Invalid operation") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, 0, 0x060E, "Invalid creation of table") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, 0, 0x060F, "No table data in memory skiplist") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, 0, 0x0610, "File already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, 0, 0x0611, "Need to reconfigure table") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, 0, 0x0612, "Invalid information to create table") +#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) //"Invalid table ID") +#define TSDB_CODE_TDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0601) //"Invalid table type") +#define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0602) //"Invalid table schema version") +#define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) //"Table already exists") +#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) //"Invalid configuration") +#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) //"Tsdb init failed") +#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) //"No diskspace for tsdb") +#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) //"No permission for disk files") +#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) //"Data file(s) corrupted") +#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) //"Out of memory") +#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) //"Tag too old") +#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) //"Timestamp data out of range") +#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) //"Submit message is messed up") +#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x060D) //"Invalid operation") +#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) //"Invalid creation of table") +#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) //"No table data in memory skiplist") +#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) //"File already exists") +#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) //"Need to reconfigure table") +#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) //"Invalid information to create table") +#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) //"No available disk") +#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) //"TSDB messed message") +#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) //"TSDB invalid tag value") // query -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "Invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, 0, 0x0701, "Invalid message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE, 0, 0x0702, "No diskspace for query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, 0, 0x0703, "System out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, 0, 0x0704, "Unexpected generic error in query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "Duplicated join key") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag conditon too many") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached") +#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") +#define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) //"Invalid message") // failed to validate the sql expression msg by vnode +#define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) //"No diskspace for query") +#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) //"System out of memory") +#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) //"Unexpected generic error in query") +#define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) //"Duplicated join key") +#define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) //"Tag conditon too many") +#define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) //"Query not ready") +#define TSDB_CODE_QRY_HAS_RSP TAOS_DEF_ERROR_CODE(0, 0x0708) //"Query should response") +#define TSDB_CODE_QRY_IN_EXEC TAOS_DEF_ERROR_CODE(0, 0x0709) //"Multiple retrieval of this query") +#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query") +#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached") +#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica") + // grant -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, 0, 0x0801, "DNode creation limited by licence") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, 0, 0x0802, "Account creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, 0, 0x0803, "Table creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, 0, 0x0804, "DB creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, 0, 0x0805, "User creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, 0, 0x0806, "Conn creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, 0, 0x0807, "Stream creation limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, 0, 0x0808, "Write speed limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, 0, 0x0809, "Storage capacity limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, 0, 0x080A, "Query time limited by license") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 0x080B, "CPU cores limited by license") +#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") +#define TSDB_CODE_GRANT_DNODE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0801) //"DNode creation limited by licence") +#define TSDB_CODE_GRANT_ACCT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0802) //"Account creation limited by license") +#define TSDB_CODE_GRANT_TIMESERIES_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0803) //"Table creation limited by license") +#define TSDB_CODE_GRANT_DB_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0804) //"DB creation limited by license") +#define TSDB_CODE_GRANT_USER_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0805) //"User creation limited by license") +#define TSDB_CODE_GRANT_CONN_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0806) //"Conn creation limited by license") +#define TSDB_CODE_GRANT_STREAM_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0807) //"Stream creation limited by license") +#define TSDB_CODE_GRANT_SPEED_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0808) //"Write speed limited by license") +#define TSDB_CODE_GRANT_STORAGE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0809) //"Storage capacity limited by license") +#define TSDB_CODE_GRANT_QUERYTIME_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080A) //"Query time limited by license") +#define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B) //"CPU cores limited by license") // sync -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "Invalid Sync Configuration") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync module not enabled") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sync version") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, 0, 0x0903, "Sync confirm expired") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TOO_MANY_FWDINFO, 0, 0x0904, "Too many sync fwd infos") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_PROTOCOL, 0, 0x0905, "Mismatched protocol") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_CLUSTERID, 0, 0x0906, "Mismatched clusterId") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_SIGNATURE, 0, 0x0907, "Mismatched signature") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CHECKSUM, 0, 0x0908, "Invalid msg checksum") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGLEN, 0, 0x0909, "Invalid msg length") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, 0, 0x090A, "Invalid msg type") +#define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) //"Invalid Sync Configuration") +#define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) //"Sync module not enabled") +#define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) //"Invalid Sync version") +#define TSDB_CODE_SYN_CONFIRM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0903) //"Sync confirm expired") +#define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) //"Too many sync fwd infos") +#define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) //"Mismatched protocol") +#define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) //"Mismatched clusterId") +#define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) //"Mismatched signature") +#define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) //"Invalid msg checksum") +#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length") +#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, 0, 0x1001, "WAL file is corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, 0, 0x1002, "WAL size exceeds limit") +#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") +#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") +#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) //"WAL size exceeds limit") // http -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, 0, 0x1101, "url is not support") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVLALID_URL, 0, 0x1102, "invalid url format") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, 0, 0x1103, "no enough memory") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUSET_TOO_BIG, 0, 0x1104, "request size is too big") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_AUTH_INFO, 0, 0x1105, "no auth info input") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_MSG_INPUT, 0, 0x1106, "request is empty") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_SQL_INPUT, 0, 0x1107, "no sql input") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_EXEC_USEDB, 0, 0x1108, "no need to execute use db cmd") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SESSION_FULL, 0, 0x1109, "session list was full") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR, 0, 0x110A, "generate taosd token error") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_MULTI_REQUEST, 0, 0x110B, "size of multi request is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, 0, 0x110C, "failed to create gzip") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, 0, 0x110D, "failed to finish gzip") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_LOGIN_FAILED, 0, 0x110E, "failed to login") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, 0, 0x1120, "invalid http version") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, 0, 0x1121, "invalid content length") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, 0, 0x1122, "invalid type of Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, 0, 0x1123, "invalid format of Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, 0, 0x1124, "invalid basic Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, 0, 0x1125, "invalid taosd Authorization") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, 0, 0x1126, "failed to parse method") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, 0, 0x1127, "failed to parse target") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, 0, 0x1128, "failed to parse http version") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, 0, 0x1129, "failed to parse sp") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, 0, 0x112A, "failed to parse status") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, 0, 0x112B, "failed to parse phrase") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, 0, 0x112C, "failed to parse crlf") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, 0, 0x112D, "failed to parse header") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, 0, 0x112E, "failed to parse header key") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, 0, 0x112F, "failed to parse header val") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, 0, 0x1130, "failed to parse chunk size") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, 0, 0x1131, "failed to parse chunk") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, 0, 0x1132, "failed to parse end section") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, 0, 0x1134, "invalid parse state") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, 0, 0x1135, "failed to parse error section") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_NULL, 0, 0x1150, "query size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_SIZE, 0, 0x1151, "query size can not more than 100") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR, 0, 0x1152, "parse grafana json error") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_NOT_INPUT, 0, 0x1160, "database name can not be null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_TOO_LONG, 0, 0x1161, "database name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_INVALID_JSON, 0, 0x1162, "invalid telegraf json fromat") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_NULL, 0, 0x1163, "metrics size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_SIZE, 0, 0x1164, "metrics size can not more than 1K") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NULL, 0, 0x1165, "metric name not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_TYPE, 0, 0x1166, "metric name type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_NULL, 0, 0x1167, "metric name length is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_LONG, 0, 0x1168, "metric name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_NULL, 0, 0x1169, "timestamp not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE, 0, 0x116A, "timestamp type should be integer") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL, 0, 0x116B, "timestamp value smaller than 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_NULL, 0, 0x116C, "tags not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, 0, 0x116D, "tags size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, 0, 0x116E, "tags size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NULL, 0, 0x116F, "tag is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_NULL, 0, 0x1170, "tag name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_SIZE, 0, 0x1171, "tag name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE, 0, 0x1172, "tag value type should be number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_NULL, 0, 0x1173, "tag value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_NULL, 0, 0x1174, "table is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_SIZE, 0, 0x1175, "table name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_NULL, 0, 0x1176, "fields not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_0, 0, 0x1177, "fields size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG, 0, 0x1178, "fields size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NULL, 0, 0x1179, "field is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_NULL, 0, 0x117A, "field name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE, 0, 0x117B, "field name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE, 0, 0x117C, "field value type should be number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL, 0, 0x117D, "field value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_HOST_NOT_STRING, 0, 0x117E, "host type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST, 0, 0x117F, "stable not exist") - -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_NOT_INPUT, 0, 0x1190, "database name can not be null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_TOO_LONG, 0, 0x1191, "database name too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_INVALID_JSON, 0, 0x1192, "invalid opentsdb json fromat") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_NULL, 0, 0x1193, "metrics size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_SIZE, 0, 0x1194, "metrics size can not more than 10K") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NULL, 0, 0x1195, "metric name not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_TYPE, 0, 0x1196, "metric name type should be string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_NULL, 0, 0x1197, "metric name length is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_LONG, 0, 0x1198, "metric name length can not more than 22") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_NULL, 0, 0x1199, "timestamp not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE, 0, 0x119A, "timestamp type should be integer") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL, 0, 0x119B, "timestamp value smaller than 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_NULL, 0, 0x119C, "tags not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_0, 0, 0x119D, "tags size is 0") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG, 0, 0x119E, "tags size too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NULL, 0, 0x119F, "tag is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_NULL, 0, 0x11A0, "tag name is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_SIZE, 0, 0x11A1, "tag name length too long") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE, 0, 0x11A2, "tag value type should be boolean, number or string") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_NULL, 0, 0x11A3, "tag value is null") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value can not more than 64") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find") -TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string") +#define TSDB_CODE_HTTP_SERVER_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x1100) //"http server is not onlin") +#define TSDB_CODE_HTTP_UNSUPPORT_URL TAOS_DEF_ERROR_CODE(0, 0x1101) //"url is not support") +#define TSDB_CODE_HTTP_INVALID_URL TAOS_DEF_ERROR_CODE(0, 0x1102) //invalid url format") +#define TSDB_CODE_HTTP_NO_ENOUGH_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1103) //"no enough memory") +#define TSDB_CODE_HTTP_REQUSET_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x1104) //"request size is too big") +#define TSDB_CODE_HTTP_NO_AUTH_INFO TAOS_DEF_ERROR_CODE(0, 0x1105) //"no auth info input") +#define TSDB_CODE_HTTP_NO_MSG_INPUT TAOS_DEF_ERROR_CODE(0, 0x1106) //"request is empty") +#define TSDB_CODE_HTTP_NO_SQL_INPUT TAOS_DEF_ERROR_CODE(0, 0x1107) //"no sql input") +#define TSDB_CODE_HTTP_NO_EXEC_USEDB TAOS_DEF_ERROR_CODE(0, 0x1108) //"no need to execute use db cmd") +#define TSDB_CODE_HTTP_SESSION_FULL TAOS_DEF_ERROR_CODE(0, 0x1109) //"session list was full") +#define TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR TAOS_DEF_ERROR_CODE(0, 0x110A) //"generate taosd token error") +#define TSDB_CODE_HTTP_INVALID_MULTI_REQUEST TAOS_DEF_ERROR_CODE(0, 0x110B) //"size of multi request is 0") +#define TSDB_CODE_HTTP_CREATE_GZIP_FAILED TAOS_DEF_ERROR_CODE(0, 0x110C) //"failed to create gzip") +#define TSDB_CODE_HTTP_FINISH_GZIP_FAILED TAOS_DEF_ERROR_CODE(0, 0x110D) //"failed to finish gzip") +#define TSDB_CODE_HTTP_LOGIN_FAILED TAOS_DEF_ERROR_CODE(0, 0x110E) //"failed to login") + +#define TSDB_CODE_HTTP_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x1120) //"invalid http version") +#define TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH TAOS_DEF_ERROR_CODE(0, 0x1121) //"invalid content length") +#define TSDB_CODE_HTTP_INVALID_AUTH_TYPE TAOS_DEF_ERROR_CODE(0, 0x1122) //"invalid type of Authorization") +#define TSDB_CODE_HTTP_INVALID_AUTH_FORMAT TAOS_DEF_ERROR_CODE(0, 0x1123) //"invalid format of Authorization") +#define TSDB_CODE_HTTP_INVALID_BASIC_AUTH TAOS_DEF_ERROR_CODE(0, 0x1124) //"invalid basic Authorization") +#define TSDB_CODE_HTTP_INVALID_TAOSD_AUTH TAOS_DEF_ERROR_CODE(0, 0x1125) //"invalid taosd Authorization") +#define TSDB_CODE_HTTP_PARSE_METHOD_FAILED TAOS_DEF_ERROR_CODE(0, 0x1126) //"failed to parse method") +#define TSDB_CODE_HTTP_PARSE_TARGET_FAILED TAOS_DEF_ERROR_CODE(0, 0x1127) //"failed to parse target") +#define TSDB_CODE_HTTP_PARSE_VERSION_FAILED TAOS_DEF_ERROR_CODE(0, 0x1128) //"failed to parse http version") +#define TSDB_CODE_HTTP_PARSE_SP_FAILED TAOS_DEF_ERROR_CODE(0, 0x1129) //"failed to parse sp") +#define TSDB_CODE_HTTP_PARSE_STATUS_FAILED TAOS_DEF_ERROR_CODE(0, 0x112A) //"failed to parse status") +#define TSDB_CODE_HTTP_PARSE_PHRASE_FAILED TAOS_DEF_ERROR_CODE(0, 0x112B) //"failed to parse phrase") +#define TSDB_CODE_HTTP_PARSE_CRLF_FAILED TAOS_DEF_ERROR_CODE(0, 0x112C) //"failed to parse crlf") +#define TSDB_CODE_HTTP_PARSE_HEADER_FAILED TAOS_DEF_ERROR_CODE(0, 0x112D) //"failed to parse header") +#define TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED TAOS_DEF_ERROR_CODE(0, 0x112E) //"failed to parse header key") +#define TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED TAOS_DEF_ERROR_CODE(0, 0x112F) //"failed to parse header val") +#define TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED TAOS_DEF_ERROR_CODE(0, 0x1130) //"failed to parse chunk size") +#define TSDB_CODE_HTTP_PARSE_CHUNK_FAILED TAOS_DEF_ERROR_CODE(0, 0x1131) //"failed to parse chunk") +#define TSDB_CODE_HTTP_PARSE_END_FAILED TAOS_DEF_ERROR_CODE(0, 0x1132) //"failed to parse end section") +#define TSDB_CODE_HTTP_PARSE_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x1134) //"invalid parse state") +#define TSDB_CODE_HTTP_PARSE_ERROR_STATE TAOS_DEF_ERROR_CODE(0, 0x1135) //"failed to parse error section") + +#define TSDB_CODE_HTTP_GC_QUERY_NULL TAOS_DEF_ERROR_CODE(0, 0x1150) //"query size is 0") +#define TSDB_CODE_HTTP_GC_QUERY_SIZE TAOS_DEF_ERROR_CODE(0, 0x1151) //"query size can not more than 100") +#define TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR TAOS_DEF_ERROR_CODE(0, 0x1152) //"parse grafana json error") + +#define TSDB_CODE_HTTP_TG_DB_NOT_INPUT TAOS_DEF_ERROR_CODE(0, 0x1160) //"database name can not be null") +#define TSDB_CODE_HTTP_TG_DB_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x1161) //"database name too long") +#define TSDB_CODE_HTTP_TG_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x1162) //"invalid telegraf json fromat") +#define TSDB_CODE_HTTP_TG_METRICS_NULL TAOS_DEF_ERROR_CODE(0, 0x1163) //"metrics size is 0") +#define TSDB_CODE_HTTP_TG_METRICS_SIZE TAOS_DEF_ERROR_CODE(0, 0x1164) //"metrics size can not more than 1K") +#define TSDB_CODE_HTTP_TG_METRIC_NULL TAOS_DEF_ERROR_CODE(0, 0x1165) //"metric name not find") +#define TSDB_CODE_HTTP_TG_METRIC_TYPE TAOS_DEF_ERROR_CODE(0, 0x1166) //"metric name type should be string") +#define TSDB_CODE_HTTP_TG_METRIC_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1167) //"metric name length is 0") +#define TSDB_CODE_HTTP_TG_METRIC_NAME_LONG TAOS_DEF_ERROR_CODE(0, 0x1168) //"metric name length too long") +#define TSDB_CODE_HTTP_TG_TIMESTAMP_NULL TAOS_DEF_ERROR_CODE(0, 0x1169) //"timestamp not find") +#define TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE TAOS_DEF_ERROR_CODE(0, 0x116A) //"timestamp type should be integer") +#define TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL TAOS_DEF_ERROR_CODE(0, 0x116B) //"timestamp value smaller than 0") +#define TSDB_CODE_HTTP_TG_TAGS_NULL TAOS_DEF_ERROR_CODE(0, 0x116C) //"tags not find") +#define TSDB_CODE_HTTP_TG_TAGS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x116D) //"tags size is 0") +#define TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x116E) //"tags size too long") +#define TSDB_CODE_HTTP_TG_TAG_NULL TAOS_DEF_ERROR_CODE(0, 0x116F) //"tag is null") +#define TSDB_CODE_HTTP_TG_TAG_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1170) //"tag name is null") +#define TSDB_CODE_HTTP_TG_TAG_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x1171) //"tag name length too long") +#define TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x1172) //"tag value type should be number or string") +#define TSDB_CODE_HTTP_TG_TAG_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x1173) //"tag value is null") +#define TSDB_CODE_HTTP_TG_TABLE_NULL TAOS_DEF_ERROR_CODE(0, 0x1174) //"table is null") +#define TSDB_CODE_HTTP_TG_TABLE_SIZE TAOS_DEF_ERROR_CODE(0, 0x1175) //"table name length too long") +#define TSDB_CODE_HTTP_TG_FIELDS_NULL TAOS_DEF_ERROR_CODE(0, 0x1176) //"fields not find") +#define TSDB_CODE_HTTP_TG_FIELDS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x1177) //"fields size is 0") +#define TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x1178) //"fields size too long") +#define TSDB_CODE_HTTP_TG_FIELD_NULL TAOS_DEF_ERROR_CODE(0, 0x1179) //"field is null") +#define TSDB_CODE_HTTP_TG_FIELD_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x117A) //"field name is null") +#define TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x117B) //"field name length too long") +#define TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x117C) //"field value type should be number or string") +#define TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x117D) //"field value is null") +#define TSDB_CODE_HTTP_TG_HOST_NOT_STRING TAOS_DEF_ERROR_CODE(0, 0x117E) //"host type should be string") +#define TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x117F) //"stable not exist") + +#define TSDB_CODE_HTTP_OP_DB_NOT_INPUT TAOS_DEF_ERROR_CODE(0, 0x1190) //"database name can not be null") +#define TSDB_CODE_HTTP_OP_DB_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x1191) //"database name too long") +#define TSDB_CODE_HTTP_OP_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x1192) //"invalid opentsdb json fromat") +#define TSDB_CODE_HTTP_OP_METRICS_NULL TAOS_DEF_ERROR_CODE(0, 0x1193) //"metrics size is 0") +#define TSDB_CODE_HTTP_OP_METRICS_SIZE TAOS_DEF_ERROR_CODE(0, 0x1194) //"metrics size can not more than 10K") +#define TSDB_CODE_HTTP_OP_METRIC_NULL TAOS_DEF_ERROR_CODE(0, 0x1195) //"metric name not find") +#define TSDB_CODE_HTTP_OP_METRIC_TYPE TAOS_DEF_ERROR_CODE(0, 0x1196) //"metric name type should be string") +#define TSDB_CODE_HTTP_OP_METRIC_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x1197) //"metric name length is 0") +#define TSDB_CODE_HTTP_OP_METRIC_NAME_LONG TAOS_DEF_ERROR_CODE(0, 0x1198) //"metric name length can not more than 22") +#define TSDB_CODE_HTTP_OP_TIMESTAMP_NULL TAOS_DEF_ERROR_CODE(0, 0x1199) //"timestamp not find") +#define TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE TAOS_DEF_ERROR_CODE(0, 0x119A) //"timestamp type should be integer") +#define TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL TAOS_DEF_ERROR_CODE(0, 0x119B) //"timestamp value smaller than 0") +#define TSDB_CODE_HTTP_OP_TAGS_NULL TAOS_DEF_ERROR_CODE(0, 0x119C) //"tags not find") +#define TSDB_CODE_HTTP_OP_TAGS_SIZE_0 TAOS_DEF_ERROR_CODE(0, 0x119D) //"tags size is 0") +#define TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG TAOS_DEF_ERROR_CODE(0, 0x119E) //"tags size too long") +#define TSDB_CODE_HTTP_OP_TAG_NULL TAOS_DEF_ERROR_CODE(0, 0x119F) //"tag is null") +#define TSDB_CODE_HTTP_OP_TAG_NAME_NULL TAOS_DEF_ERROR_CODE(0, 0x11A0) //"tag name is null") +#define TSDB_CODE_HTTP_OP_TAG_NAME_SIZE TAOS_DEF_ERROR_CODE(0, 0x11A1) //"tag name length too long") +#define TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x11A2) //"tag value type should be boolean number or string") +#define TSDB_CODE_HTTP_OP_TAG_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x11A3) //"tag value is null") +#define TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x11A4) //"tag value can not more than 64") +#define TSDB_CODE_HTTP_OP_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x11A5) //"value not find") +#define TSDB_CODE_HTTP_OP_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x11A6) //"value type should be boolean number or string") // odbc -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, 0, 0x2100, "out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, 0, 0x2101, "convertion not a valid literal input") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, 0, 0x2102, "convertion undefined") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC_FRAC, 0, 0x2103, "convertion fractional truncated") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC, 0, 0x2104, "convertion truncated") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_SUPPORT, 0, 0x2105, "convertion not supported") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_OOR, 0, 0x2106, "convertion numeric value out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OUT_OF_RANGE, 0, 0x2107, "out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NOT_SUPPORT, 0, 0x2108, "not supported yet") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_HANDLE, 0, 0x2109, "invalid handle") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_RESULT, 0, 0x210a, "no result set") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_FIELDS, 0, 0x210b, "no fields returned") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_CURSOR, 0, 0x210c, "invalid cursor") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_STATEMENT_NOT_READY, 0, 0x210d, "statement not ready") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONNECTION_BUSY, 0, 0x210e, "connection still busy") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_CONNSTR, 0, 0x210f, "bad connection string") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_ARG, 0, 0x2110, "bad argument") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_VALID_TS, 0, 0x2111, "not a valid timestamp") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE, 0, 0x2112, "src too large") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, 0, 0x2113, "src bad sequence") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, 0, 0x2114, "src incomplete") -TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, 0, 0x2115, "src general") +#define TSDB_CODE_ODBC_OOM TAOS_DEF_ERROR_CODE(0, 0x2100) //"out of memory") +#define TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM TAOS_DEF_ERROR_CODE(0, 0x2101) //"convertion not a valid literal input") +#define TSDB_CODE_ODBC_CONV_UNDEF TAOS_DEF_ERROR_CODE(0, 0x2102) //"convertion undefined") +#define TSDB_CODE_ODBC_CONV_TRUNC_FRAC TAOS_DEF_ERROR_CODE(0, 0x2103) //"convertion fractional truncated") +#define TSDB_CODE_ODBC_CONV_TRUNC TAOS_DEF_ERROR_CODE(0, 0x2104) //"convertion truncated") +#define TSDB_CODE_ODBC_CONV_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x2105) //"convertion not supported") +#define TSDB_CODE_ODBC_CONV_OOR TAOS_DEF_ERROR_CODE(0, 0x2106) //"convertion numeric value out of range") +#define TSDB_CODE_ODBC_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x2107) //"out of range") +#define TSDB_CODE_ODBC_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x2108) //"not supported yet") +#define TSDB_CODE_ODBC_INVALID_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2109) //"invalid handle") +#define TSDB_CODE_ODBC_NO_RESULT TAOS_DEF_ERROR_CODE(0, 0x210a) //"no result set") +#define TSDB_CODE_ODBC_NO_FIELDS TAOS_DEF_ERROR_CODE(0, 0x210b) //"no fields returned") +#define TSDB_CODE_ODBC_INVALID_CURSOR TAOS_DEF_ERROR_CODE(0, 0x210c) //"invalid cursor") +#define TSDB_CODE_ODBC_STATEMENT_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x210d) //"statement not ready") +#define TSDB_CODE_ODBC_CONNECTION_BUSY TAOS_DEF_ERROR_CODE(0, 0x210e) //"connection still busy") +#define TSDB_CODE_ODBC_BAD_CONNSTR TAOS_DEF_ERROR_CODE(0, 0x210f) //"bad connection string") +#define TSDB_CODE_ODBC_BAD_ARG TAOS_DEF_ERROR_CODE(0, 0x2110) //"bad argument") +#define TSDB_CODE_ODBC_CONV_NOT_VALID_TS TAOS_DEF_ERROR_CODE(0, 0x2111) //"not a valid timestamp") +#define TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x2112) //"src too large") +#define TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ TAOS_DEF_ERROR_CODE(0, 0x2113) //"src bad sequence") +#define TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE TAOS_DEF_ERROR_CODE(0, 0x2114) //"src incomplete") +#define TSDB_CODE_ODBC_CONV_SRC_GENERAL TAOS_DEF_ERROR_CODE(0, 0x2115) //"src general") // tfs -TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY, 0, 0x2200, "tfs out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, 0, 0x2201, "tfs invalid mount config") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT, 0, 0x2202, "tfs too many mount") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY, 0, 0x2203, "tfs duplicate primary mount") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_PRIMARY_DISK, 0, 0x2204, "tfs no primary mount") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_MOUNT_AT_TIER, 0, 0x2205, "tfs no mount at tier") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, 0, 0x2206, "tfs file already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, 0, 0x2207, "tfs invalid level") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, 0, 0x2208, "tfs no valid disk") - - -#ifdef TAOS_ERROR_C -}; -#endif - +#define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") +#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) //"tfs invalid mount config") +#define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) //"tfs too many mount") +#define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) //"tfs duplicate primary mount") +#define TSDB_CODE_FS_NO_PRIMARY_DISK TAOS_DEF_ERROR_CODE(0, 0x2204) //"tfs no primary mount") +#define TSDB_CODE_FS_NO_MOUNT_AT_TIER TAOS_DEF_ERROR_CODE(0, 0x2205) //"tfs no mount at tier") +#define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) //"tfs file already exists") +#define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) //"tfs invalid level") +#define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) //"tfs no valid disk") #ifdef __cplusplus } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 905867fbc7355b34f87e62ab71f8172e4b7bf93e..99f4e1518f1e1536daa30a4b0c8b161acadf5bc3 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -59,6 +59,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_DROP_STABLE, "drop-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_STREAM, "alter-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CONFIG_DNODE, "config-dnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_ALTER_VNODE, "alter-vnode" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_SYNC_VNODE, "sync-vnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MD_CREATE_MNODE, "create-mnode" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY6, "dummy6" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY7, "dummy7" ) @@ -77,6 +78,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DB, "create-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_DB, "alter-db" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_SYNC_DB, "sync-db-replica" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TABLE, "create-table" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_TABLE, "drop-table" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_TABLE, "alter-table" ) @@ -107,6 +109,12 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY13, "dummy13" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY14, "dummy14" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" ) +// message for topic +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TP, "create-tp" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_TP, "drop-tp" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_TP, "use-tp" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_TP, "alter-tp" ) + #ifndef TAOS_MESSAGE_C TSDB_MSG_TYPE_MAX // 105 #endif @@ -141,6 +149,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_VNODES, TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_CLUSTER, + TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_MAX, }; @@ -153,30 +162,31 @@ enum _mgmt_table { #define TSDB_ALTER_TABLE_DROP_COLUMN 6 #define TSDB_ALTER_TABLE_CHANGE_COLUMN 7 -#define TSDB_FILL_NONE 0 -#define TSDB_FILL_NULL 1 -#define TSDB_FILL_SET_VALUE 2 -#define TSDB_FILL_LINEAR 3 -#define TSDB_FILL_PREV 4 +#define TSDB_FILL_NONE 0 +#define TSDB_FILL_NULL 1 +#define TSDB_FILL_SET_VALUE 2 +#define TSDB_FILL_LINEAR 3 +#define TSDB_FILL_PREV 4 +#define TSDB_FILL_NEXT 5 -#define TSDB_ALTER_USER_PASSWD 0x1 +#define TSDB_ALTER_USER_PASSWD 0x1 #define TSDB_ALTER_USER_PRIVILEGES 0x2 -#define TSDB_KILL_MSG_LEN 30 +#define TSDB_KILL_MSG_LEN 30 -#define TSDB_VN_READ_ACCCESS ((char)0x1) -#define TSDB_VN_WRITE_ACCCESS ((char)0x2) +#define TSDB_VN_READ_ACCCESS ((char)0x1) +#define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u // the normal column of the table -#define TSDB_COL_TAG 0x1u // the tag column type -#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column -#define TSDB_COL_NULL 0x4u // the column filter NULL or not +#define TSDB_COL_NORMAL 0x0u // the normal column of the table +#define TSDB_COL_TAG 0x1u // the tag column type +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column +#define TSDB_COL_NULL 0x4u // the column filter NULL or not -#define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) -#define TSDB_COL_IS_NORMAL_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) -#define TSDB_COL_IS_UD_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_UDC) -#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) +#define TSDB_COL_IS_TAG(f) (((f&(~(TSDB_COL_NULL)))&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f&(~(TSDB_COL_NULL))) == TSDB_COL_UDC) +#define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) extern char *taosMsg[]; @@ -260,15 +270,14 @@ typedef struct { uint64_t uid; uint64_t superTableUid; uint64_t createdTime; - char tableId[TSDB_TABLE_FNAME_LEN]; - char superTableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; + char stableFname[TSDB_TABLE_FNAME_LEN]; char data[]; } SMDCreateTableMsg; typedef struct { int32_t len; // one create table message - char tableId[TSDB_TABLE_FNAME_LEN]; - char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_FNAME_LEN]; int8_t igExists; int8_t getMeta; int16_t numOfTags; @@ -284,12 +293,12 @@ typedef struct { } SCMCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_FNAME_LEN]; + char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; } SCMDropTableMsg; typedef struct { - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; int16_t type; /* operation type */ int16_t numOfCols; /* number of schema */ @@ -369,25 +378,25 @@ typedef struct { int32_t vgId; int32_t tid; uint64_t uid; - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; } SMDDropTableMsg; typedef struct { int32_t contLen; int32_t vgId; uint64_t uid; - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; } SDropSTableMsg; typedef struct { int32_t vgId; -} SDropVnodeMsg; +} SDropVnodeMsg, SSyncVnodeMsg; typedef struct SColIndex { int16_t colId; // column id int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag uint16_t flag; // denote if it is a tag or a normal column - char name[TSDB_COL_NAME_LEN]; + char name[TSDB_COL_NAME_LEN]; // TODO remove it } SColIndex; /* sql function msg, to describe the message to vnode about sql function @@ -395,7 +404,10 @@ typedef struct SColIndex { typedef struct SSqlFuncMsg { int16_t functionId; int16_t numOfParams; + int16_t resColId; // result column id, id of the current output column + int16_t colType; + int16_t colBytes; SColIndex colInfo; struct ArgElem { @@ -475,12 +487,13 @@ typedef struct { int16_t orderColId; int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; + SSessionWindow sw; // session window uint16_t tagCondLen; // tag length in current query uint32_t tbnameCondLen; // table name filter condition string length int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx - int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. + int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. int16_t prjOrder; // global order in super table projection query. int64_t limit; int64_t offset; @@ -496,17 +509,19 @@ typedef struct { int32_t tsOrder; // ts comp block order int32_t numOfTags; // number of tags columns involved int32_t sqlstrLen; // sql query string + int32_t prevResultLen; // previous result length SColumnInfo colList[]; } SQueryTableMsg; typedef struct { int32_t code; - uint64_t qhandle; // query handle + union{uint64_t qhandle; uint64_t qId;}; // query handle } SQueryTableRsp; +// todo: the show handle should be replaced with id typedef struct { SMsgHead header; - uint64_t qhandle; + union{uint64_t qhandle; uint64_t qId;}; // query handle uint16_t free; } SRetrieveTableMsg; @@ -554,13 +569,15 @@ typedef struct { int8_t ignoreExist; int8_t update; int8_t cacheLastRow; - int8_t reserve[8]; + int8_t dbType; + int16_t partitions; + int8_t reserve[5]; } SCreateDbMsg, SAlterDbMsg; typedef struct { char db[TSDB_TABLE_FNAME_LEN]; uint8_t ignoreNotExists; -} SDropDbMsg, SUseDbMsg; +} SDropDbMsg, SUseDbMsg, SSyncDbMsg; // IMPORTANT: sizeof(SVnodeStatisticInfo) should not exceed // TSDB_FILE_HEADER_LEN/4 - TSDB_FILE_HEADER_VERSION_SIZE @@ -615,6 +632,7 @@ typedef struct { int32_t maxtablesPerVnode; int32_t maxVgroupsPerDb; char arbitrator[TSDB_EP_LEN]; // tsArbitrator + char reserve[2]; // to solve arm32 bus error char timezone[64]; // tsTimezone int64_t checkTime; // 1970-01-01 00:00:00.000 char locale[TSDB_LOCALE_LEN]; // tsLocale @@ -673,7 +691,8 @@ typedef struct { int8_t cacheLastRow; int32_t vgCfgVersion; int8_t dbReplica; - int8_t reserved[9]; + int8_t dbType; + int8_t reserved[8]; } SVnodeCfg; typedef struct { @@ -688,7 +707,7 @@ typedef struct { } SCreateVnodeMsg, SAlterVnodeMsg; typedef struct { - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; int16_t createFlag; char tags[]; } STableInfoMsg; @@ -726,7 +745,7 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_FNAME_LEN]; // table id + char tableFname[TSDB_TABLE_FNAME_LEN]; // table id uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -847,7 +866,7 @@ typedef struct { uint64_t uid; uint64_t stime; // stream starting time int32_t status; - char tableId[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; } SAlterStreamMsg; typedef struct { diff --git a/src/inc/tfs.h b/src/inc/tfs.h new file mode 100644 index 0000000000000000000000000000000000000000..4ed21bc6e1ddb4d9011d701ad99e6f2a71f54b0e --- /dev/null +++ b/src/inc/tfs.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TD_TFS_H +#define TD_TFS_H + +#include "tglobal.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + int level; + int id; +} SDiskID; + +#define TFS_UNDECIDED_LEVEL -1 +#define TFS_UNDECIDED_ID -1 +#define TFS_PRIMARY_LEVEL 0 +#define TFS_PRIMARY_ID 0 + +// FS APIs ==================================== +typedef struct { + int64_t tsize; + int64_t used; + int64_t avail; +} SFSMeta; + +int tfsInit(SDiskCfg *pDiskCfg, int ndisk); +void tfsDestroy(); +void tfsUpdateInfo(SFSMeta *pFSMeta); +void tfsGetMeta(SFSMeta *pMeta); +void tfsAllocDisk(int expLevel, int *level, int *id); + +const char *TFS_PRIMARY_PATH(); +const char *TFS_DISK_PATH(int level, int id); + +// TFILE APIs ==================================== +typedef struct { + int level; + int id; + char rname[TSDB_FILENAME_LEN]; // REL name + char aname[TSDB_FILENAME_LEN]; // ABS name +} TFILE; + +#define TFILE_LEVEL(pf) ((pf)->level) +#define TFILE_ID(pf) ((pf)->id) +#define TFILE_NAME(pf) ((pf)->aname) +#define TFILE_REL_NAME(pf) ((pf)->rname) + +#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags) +#define tfsclose(fd) close(fd) +#define tfsremove(pf) remove(TFILE_NAME(pf)) +#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df)) +#define tfsrename(sf, df) taosRename(TFILE_NAME(sf), TFILE_NAME(df)) + +void tfsInitFile(TFILE *pf, int level, int id, const char *bname); +bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2); +int tfsEncodeFile(void **buf, TFILE *pf); +void *tfsDecodeFile(void *buf, TFILE *pf); +void tfsbasename(const TFILE *pf, char *dest); +void tfsdirname(const TFILE *pf, char *dest); + +// DIR APIs ==================================== +int tfsMkdirAt(const char *rname, int level, int id); +int tfsMkdirRecurAt(const char *rname, int level, int id); +int tfsMkdir(const char *rname); +int tfsRmdir(const char *rname); +int tfsRename(char *orname, char *nrname); + +typedef struct TDIR TDIR; + +TDIR * tfsOpendir(const char *rname); +const TFILE *tfsReaddir(TDIR *tdir); +void tfsClosedir(TDIR *tdir); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/common/inc/tsystem.h b/src/inc/tp.h similarity index 62% rename from src/common/inc/tsystem.h rename to src/inc/tp.h index 93d305e49c6707e7c567a2fbca94b652067f421a..1d6570898ebfcf0137fd2d502dc39de9378714b9 100644 --- a/src/common/inc/tsystem.h +++ b/src/inc/tp.h @@ -13,23 +13,16 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSYSTEM_H -#define TDENGINE_TSYSTEM_H +#ifndef TDENGINE_TP +#define TDENGINE_TP #ifdef __cplusplus extern "C" { #endif -bool taosGetSysMemory(float *memoryUsedMB); -bool taosGetProcMemory(float *memoryUsedMB); -bool taosGetDisk(); -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); -bool taosGetBandSpeed(float *bandSpeedKb); -bool taosGetProcIO(float *readKB, float *writeKB); -void taosGetSystemInfo(); -void taosPrintOsInfo(); -void taosKillSystem(); -void taosSetCoreDump(); +int32_t tpInit(); +void tpCleanUp(); +void tpUpdateTs(int32_t vgId, int64_t *seq, void *pMsg); #ifdef __cplusplus } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 262bf30309ae8bc8bb24b844a88acbfd2290d0c0..495bfa2384ae0680577775c83f63d098ebf4585f 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -25,6 +25,8 @@ #include "tdataformat.h" #include "tname.h" #include "hash.h" +#include "tlockfree.h" +#include "tlist.h" #ifdef __cplusplus extern "C" { @@ -40,7 +42,8 @@ extern "C" { // TSDB STATE DEFINITION #define TSDB_STATE_OK 0x0 -#define TSDB_STATE_BAD_FILE 0x1 +#define TSDB_STATE_BAD_META 0x1 +#define TSDB_STATE_BAD_DATA 0x2 // --------- TSDB APPLICATION HANDLE DEFINITION typedef struct { @@ -48,7 +51,7 @@ typedef struct { void *cqH; int (*notifyStatus)(void *, int status, int eno); int (*eventCallBack)(void *); - void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema); + void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema); void (*cqDropFunc)(void *handle); } STsdbAppH; @@ -76,17 +79,17 @@ typedef struct { int64_t pointsWritten; // total data points written } STsdbStat; -typedef void TSDB_REPO_T; // use void to hide implementation details from outside +typedef struct STsdbRepo STsdbRepo; -STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo); +STsdbCfg *tsdbGetCfg(const STsdbRepo *repo); // --------- TSDB REPOSITORY DEFINITION -int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg); -int32_t tsdbDropRepo(char *rootDir); -TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); -int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); -int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg); -int tsdbGetState(TSDB_REPO_T *repo); +int32_t tsdbCreateRepo(int repoid); +int32_t tsdbDropRepo(int repoid); +STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); +int tsdbCloseRepo(STsdbRepo *repo, int toCommit); +int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg); +int tsdbGetState(STsdbRepo *repo); // --------- TSDB TABLE DEFINITION typedef struct { @@ -110,19 +113,20 @@ typedef struct { void tsdbClearTableCfg(STableCfg *config); -void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes); -char* tsdbGetTableName(void *pTable); +void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes); +char *tsdbGetTableName(void *pTable); #define TSDB_TABLEID(_table) ((STableId*) (_table)) +#define TSDB_PREV_ROW 0x1 +#define TSDB_NEXT_ROW 0x2 STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg); -int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg); -int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); -int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); -// TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); +int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg); +int tsdbDropTable(STsdbRepo *pRepo, STableId tableId); +int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg); -uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); +uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); // the TSDB repository info typedef struct STsdbRepoInfo { @@ -132,7 +136,7 @@ typedef struct STsdbRepoInfo { int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository // TODO: Other informations to add } STsdbRepoInfo; -STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo); +STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo); // the meter information report structure typedef struct { @@ -141,7 +145,6 @@ typedef struct { int64_t tableTotalDataSize; // In bytes int64_t tableTotalDiskSize; // In bytes } STableInfo; -STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid); // -- FOR INSERT DATA /** @@ -151,24 +154,52 @@ STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid); * * @return the number of points inserted, -1 for failure and the error number is set */ -int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp); +int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp); // -- FOR QUERY TIME SERIES DATA typedef void *TsdbQueryHandleT; // Use void to hide implementation details -// query condition to build vnode iterator +#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 +#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 +#define BLOCK_LOAD_TABLE_RR_ORDER 3 + +// query condition to build multi-table data block iterator typedef struct STsdbQueryCond { STimeWindow twindow; - int32_t order; // desc|asc order to iterate the data block + int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo *colList; + bool loadExternalRows; // load external rows or not + int32_t type; // data block load type: } STsdbQueryCond; +typedef struct STableData STableData; +typedef struct { + T_REF_DECLARE() + SRWLatch latch; + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfRows; + int32_t maxTables; + STableData **tData; + SList * actList; + SList * extraBuffList; + SList * bufBlockList; + int64_t pointsAdd; // TODO + int64_t storageAdd; // TODO +} SMemTable; + +typedef struct { + SMemTable* mem; + SMemTable* imem; + SMemTable mtable; + SMemTable* omem; +} SMemSnapshot; + typedef struct SMemRef { - int32_t ref; - void *mem; - void *imem; + int32_t ref; + SMemSnapshot snapshot; } SMemRef; typedef struct SDataBlockInfo { @@ -179,17 +210,31 @@ typedef struct SDataBlockInfo { int32_t tid; } SDataBlockInfo; +typedef struct SFileBlockInfo { + int32_t numOfRows; +} SFileBlockInfo; + typedef struct { - void *pTable; - TSKEY lastKey; + void *pTable; + TSKEY lastKey; } STableKeyInfo; typedef struct { - size_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid + uint32_t numOfTables; + SArray * pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; +typedef struct { + uint16_t rowSize; + uint16_t numOfFiles; + uint32_t numOfTables; + uint64_t totalSize; + int32_t firstSeekTimeUs; + uint32_t numOfRowsInMemTable; + SArray *dataBlockInfos; +} STableBlockDist; + /** * Get the data block iterator, starting from position according to the query condition * @@ -200,7 +245,8 @@ typedef struct { * @param qinfo query info handle from query processor * @return */ -TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef); +TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, + SMemRef *pRef); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -212,14 +258,15 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab * @param tableInfo table list. * @return */ -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef); +TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, + SMemRef *pRef); /** * get the queried table object list * @param pHandle * @return */ -SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); +SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); /** * get the group list according to table id from client @@ -229,16 +276,26 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle); * @param qinfo * @return */ -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, - void *qinfo, SMemRef* pRef); +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, + uint64_t qId, SMemRef *pRef); + /** - * move to next block if exists + * get num of rows in mem table + * + * @param pHandle + * @return row size + */ + +int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); + +/** + * move to next block if exists * * @param pQueryHandle * @return */ -bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle); +bool tsdbNextDataBlock(TsdbQueryHandleT pQueryHandle); /** * Get current data block information @@ -247,7 +304,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle); * @param pBlockInfo * @return */ -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo* pBlockInfo); +void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo); /** * @@ -278,12 +335,12 @@ SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdL * @param stableid. super table sid * @param pTagCond. tag query condition */ -int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len, int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols); /** - * destory the created table group list, which is generated by tag query + * destroy the created table group list, which is generated by tag query * @param pGroupList */ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); @@ -296,7 +353,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(STsdbRepo *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); /** * @@ -305,7 +362,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, ST * @param pGroupInfo * @return */ -int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo); +int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); /** * clean up the query handle @@ -313,6 +370,12 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa */ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle); +void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond); + +void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList); + +int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo); + /** * get the statistics of repo usage * @param repo. point to the tsdbrepo @@ -324,10 +387,14 @@ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int int tsdbInitCommitQueue(); void tsdbDestroyCommitQueue(); -int tsdbSyncCommit(TSDB_REPO_T *repo); +int tsdbSyncCommit(STsdbRepo *repo); void tsdbIncCommitRef(int vgId); void tsdbDecCommitRef(int vgId); +// For TSDB file sync +int tsdbSyncSend(void *pRepo, SOCKET socketFd); +int tsdbSyncRecv(void *pRepo, SOCKET socketFd); + #ifdef __cplusplus } #endif diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 4dae86bbed538a0801251f62723a471215655e24..d1b68e3f5a1e27b63dd08a4d1d4862c7b1e68179 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -56,16 +56,6 @@ typedef struct { int32_t role[TAOS_SYNC_MAX_REPLICA]; } SNodesRole; -/* - if name is empty(name[0] is zero), get the file from index or after, but not larger than eindex. If a file - is found between index and eindex, index shall be updated, name shall be set, size shall be set to - file size, and file magic number shall be returned. - - if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return - zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated. -*/ -typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); - // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); @@ -83,24 +73,31 @@ typedef void (*FNotifyRole)(int32_t vgId, int8_t role); typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); // when data file is synced successfully, notity app -typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); +typedef void (*FStartSyncFile)(int32_t vgId); +typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); // get file version typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); +typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); +typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); + typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version SSyncCfg syncCfg; // configuration from mgmt char path[TSDB_FILENAME_LEN]; // path to the file - FGetFileInfo getFileInfo; - FGetWalInfo getWalInfo; - FWriteToCache writeToCache; + void * pTsdb; + FGetWalInfo getWalInfoFp; + FWriteToCache writeToCacheFp; FConfirmForward confirmForward; - FNotifyRole notifyRole; - FNotifyFlowCtrl notifyFlowCtrl; - FNotifyFileSynced notifyFileSynced; - FGetVersion getVersion; + FNotifyRole notifyRoleFp; + FNotifyFlowCtrl notifyFlowCtrlFp; + FStartSyncFile startSyncFileFp; + FStopSyncFile stopSyncFileFp; + FGetVersion getVersionFp; + FSendFile sendFileFp; + FRecvFile recvFileFp; } SSyncInfo; typedef void *tsync_h; @@ -111,8 +108,8 @@ void syncCleanUp(); int64_t syncStart(const SSyncInfo *); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg *); -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype); -void syncConfirmForward(int64_t rid, uint64_t version, int32_t code); +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force); +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force); void syncRecover(int64_t rid); // recover from other nodes: int32_t syncGetNodesRole(int64_t rid, SNodesRole *); diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index 09500fc8c575c8660076e0ef5093a1590e49f90f..e9f95660f7d6defb6a8dd63b7431bd0ab913b4ab 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_TTOKENDEF_H #define TDENGINE_TTOKENDEF_H + #define TK_ID 1 #define TK_BOOL 2 #define TK_TINYINT 3 @@ -61,170 +62,147 @@ #define TK_BITNOT 43 #define TK_SHOW 44 #define TK_DATABASES 45 -#define TK_MNODES 46 -#define TK_DNODES 47 -#define TK_ACCOUNTS 48 -#define TK_USERS 49 -#define TK_MODULES 50 -#define TK_QUERIES 51 -#define TK_CONNECTIONS 52 -#define TK_STREAMS 53 -#define TK_VARIABLES 54 -#define TK_SCORES 55 -#define TK_GRANTS 56 -#define TK_VNODES 57 -#define TK_IPTOKEN 58 -#define TK_DOT 59 -#define TK_CREATE 60 -#define TK_TABLE 61 -#define TK_DATABASE 62 -#define TK_TABLES 63 -#define TK_STABLES 64 -#define TK_VGROUPS 65 -#define TK_DROP 66 -#define TK_DNODE 67 -#define TK_USER 68 -#define TK_ACCOUNT 69 -#define TK_USE 70 -#define TK_DESCRIBE 71 -#define TK_ALTER 72 -#define TK_PASS 73 -#define TK_PRIVILEGE 74 -#define TK_LOCAL 75 -#define TK_IF 76 -#define TK_EXISTS 77 -#define TK_PPS 78 -#define TK_TSERIES 79 -#define TK_DBS 80 -#define TK_STORAGE 81 -#define TK_QTIME 82 -#define TK_CONNS 83 -#define TK_STATE 84 -#define TK_KEEP 85 -#define TK_CACHE 86 -#define TK_REPLICA 87 -#define TK_QUORUM 88 -#define TK_DAYS 89 -#define TK_MINROWS 90 -#define TK_MAXROWS 91 -#define TK_BLOCKS 92 -#define TK_CTIME 93 -#define TK_WAL 94 -#define TK_FSYNC 95 -#define TK_COMP 96 -#define TK_PRECISION 97 -#define TK_UPDATE 98 -#define TK_CACHELAST 99 -#define TK_LP 100 -#define TK_RP 101 -#define TK_UNSIGNED 102 -#define TK_TAGS 103 -#define TK_USING 104 -#define TK_AS 105 -#define TK_COMMA 106 -#define TK_NULL 107 -#define TK_SELECT 108 -#define TK_UNION 109 -#define TK_ALL 110 -#define TK_FROM 111 -#define TK_VARIABLE 112 -#define TK_INTERVAL 113 -#define TK_FILL 114 -#define TK_SLIDING 115 -#define TK_ORDER 116 -#define TK_BY 117 -#define TK_ASC 118 -#define TK_DESC 119 -#define TK_GROUP 120 -#define TK_HAVING 121 -#define TK_LIMIT 122 -#define TK_OFFSET 123 -#define TK_SLIMIT 124 -#define TK_SOFFSET 125 -#define TK_WHERE 126 -#define TK_NOW 127 -#define TK_RESET 128 -#define TK_QUERY 129 -#define TK_ADD 130 -#define TK_COLUMN 131 -#define TK_TAG 132 -#define TK_CHANGE 133 -#define TK_SET 134 -#define TK_KILL 135 -#define TK_CONNECTION 136 -#define TK_STREAM 137 -#define TK_COLON 138 -#define TK_ABORT 139 -#define TK_AFTER 140 -#define TK_ATTACH 141 -#define TK_BEFORE 142 -#define TK_BEGIN 143 -#define TK_CASCADE 144 -#define TK_CLUSTER 145 -#define TK_CONFLICT 146 -#define TK_COPY 147 -#define TK_DEFERRED 148 -#define TK_DELIMITERS 149 -#define TK_DETACH 150 -#define TK_EACH 151 -#define TK_END 152 -#define TK_EXPLAIN 153 -#define TK_FAIL 154 -#define TK_FOR 155 -#define TK_IGNORE 156 -#define TK_IMMEDIATE 157 -#define TK_INITIALLY 158 -#define TK_INSTEAD 159 -#define TK_MATCH 160 -#define TK_KEY 161 -#define TK_OF 162 -#define TK_RAISE 163 -#define TK_REPLACE 164 -#define TK_RESTRICT 165 -#define TK_ROW 166 -#define TK_STATEMENT 167 -#define TK_TRIGGER 168 -#define TK_VIEW 169 -#define TK_COUNT 170 -#define TK_SUM 171 -#define TK_AVG 172 -#define TK_MIN 173 -#define TK_MAX 174 -#define TK_FIRST 175 -#define TK_LAST 176 -#define TK_TOP 177 -#define TK_BOTTOM 178 -#define TK_STDDEV 179 -#define TK_PERCENTILE 180 -#define TK_APERCENTILE 181 -#define TK_LEASTSQUARES 182 -#define TK_HISTOGRAM 183 -#define TK_DIFF 184 -#define TK_SPREAD 185 -#define TK_TWA 186 -#define TK_INTERP 187 -#define TK_LAST_ROW 188 -#define TK_RATE 189 -#define TK_IRATE 190 -#define TK_SUM_RATE 191 -#define TK_SUM_IRATE 192 -#define TK_AVG_RATE 193 -#define TK_AVG_IRATE 194 -#define TK_TBID 195 -#define TK_SEMI 196 -#define TK_NONE 197 -#define TK_PREV 198 -#define TK_LINEAR 199 -#define TK_IMPORT 200 -#define TK_METRIC 201 -#define TK_TBNAME 202 -#define TK_JOIN 203 -#define TK_METRICS 204 -#define TK_STABLE 205 -#define TK_INSERT 206 -#define TK_INTO 207 -#define TK_VALUES 208 - +#define TK_TOPICS 46 +#define TK_MNODES 47 +#define TK_DNODES 48 +#define TK_ACCOUNTS 49 +#define TK_USERS 50 +#define TK_MODULES 51 +#define TK_QUERIES 52 +#define TK_CONNECTIONS 53 +#define TK_STREAMS 54 +#define TK_VARIABLES 55 +#define TK_SCORES 56 +#define TK_GRANTS 57 +#define TK_VNODES 58 +#define TK_IPTOKEN 59 +#define TK_DOT 60 +#define TK_CREATE 61 +#define TK_TABLE 62 +#define TK_DATABASE 63 +#define TK_TABLES 64 +#define TK_STABLES 65 +#define TK_VGROUPS 66 +#define TK_DROP 67 +#define TK_STABLE 68 +#define TK_TOPIC 69 +#define TK_DNODE 70 +#define TK_USER 71 +#define TK_ACCOUNT 72 +#define TK_USE 73 +#define TK_DESCRIBE 74 +#define TK_ALTER 75 +#define TK_PASS 76 +#define TK_PRIVILEGE 77 +#define TK_LOCAL 78 +#define TK_IF 79 +#define TK_EXISTS 80 +#define TK_PPS 81 +#define TK_TSERIES 82 +#define TK_DBS 83 +#define TK_STORAGE 84 +#define TK_QTIME 85 +#define TK_CONNS 86 +#define TK_STATE 87 +#define TK_KEEP 88 +#define TK_CACHE 89 +#define TK_REPLICA 90 +#define TK_QUORUM 91 +#define TK_DAYS 92 +#define TK_MINROWS 93 +#define TK_MAXROWS 94 +#define TK_BLOCKS 95 +#define TK_CTIME 96 +#define TK_WAL 97 +#define TK_FSYNC 98 +#define TK_COMP 99 +#define TK_PRECISION 100 +#define TK_UPDATE 101 +#define TK_CACHELAST 102 +#define TK_PARTITIONS 103 +#define TK_LP 104 +#define TK_RP 105 +#define TK_UNSIGNED 106 +#define TK_TAGS 107 +#define TK_USING 108 +#define TK_COMMA 109 +#define TK_AS 110 +#define TK_NULL 111 +#define TK_SELECT 112 +#define TK_UNION 113 +#define TK_ALL 114 +#define TK_DISTINCT 115 +#define TK_FROM 116 +#define TK_VARIABLE 117 +#define TK_INTERVAL 118 +#define TK_SESSION 119 +#define TK_FILL 120 +#define TK_SLIDING 121 +#define TK_ORDER 122 +#define TK_BY 123 +#define TK_ASC 124 +#define TK_DESC 125 +#define TK_GROUP 126 +#define TK_HAVING 127 +#define TK_LIMIT 128 +#define TK_OFFSET 129 +#define TK_SLIMIT 130 +#define TK_SOFFSET 131 +#define TK_WHERE 132 +#define TK_NOW 133 +#define TK_RESET 134 +#define TK_QUERY 135 +#define TK_SYNCDB 136 +#define TK_ADD 137 +#define TK_COLUMN 138 +#define TK_TAG 139 +#define TK_CHANGE 140 +#define TK_SET 141 +#define TK_KILL 142 +#define TK_CONNECTION 143 +#define TK_STREAM 144 +#define TK_COLON 145 +#define TK_ABORT 146 +#define TK_AFTER 147 +#define TK_ATTACH 148 +#define TK_BEFORE 149 +#define TK_BEGIN 150 +#define TK_CASCADE 151 +#define TK_CLUSTER 152 +#define TK_CONFLICT 153 +#define TK_COPY 154 +#define TK_DEFERRED 155 +#define TK_DELIMITERS 156 +#define TK_DETACH 157 +#define TK_EACH 158 +#define TK_END 159 +#define TK_EXPLAIN 160 +#define TK_FAIL 161 +#define TK_FOR 162 +#define TK_IGNORE 163 +#define TK_IMMEDIATE 164 +#define TK_INITIALLY 165 +#define TK_INSTEAD 166 +#define TK_MATCH 167 +#define TK_KEY 168 +#define TK_OF 169 +#define TK_RAISE 170 +#define TK_REPLACE 171 +#define TK_RESTRICT 172 +#define TK_ROW 173 +#define TK_STATEMENT 174 +#define TK_TRIGGER 175 +#define TK_VIEW 176 +#define TK_SEMI 177 +#define TK_NONE 178 +#define TK_PREV 179 +#define TK_LINEAR 180 +#define TK_IMPORT 181 +#define TK_TBNAME 182 +#define TK_JOIN 183 +#define TK_INSERT 184 +#define TK_INTO 185 +#define TK_VALUES 186 #define TK_SPACE 300 diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 1849139df12c2970ef554a722411fec1131262aa..662a23bfdbc52e432d2bcfbdac5c43cfba5d60dc 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -7,6 +7,28 @@ extern "C" { #include "taosdef.h" +// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR +typedef int32_t VarDataOffsetT; +typedef int16_t VarDataLenT; + +typedef struct tstr { + VarDataLenT len; + char data[]; +} tstr; + +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) +#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) + +// this data type is internally used only in 'in' query to hold the values +#define TSDB_DATA_TYPE_ARRAY (1000) + #define GET_TYPED_DATA(_v, _finalType, _type, _data) \ do { \ switch (_type) { \ @@ -23,6 +45,7 @@ extern "C" { case TSDB_DATA_TYPE_USMALLINT: \ (_v) = (_finalType)GET_UINT16_VAL(_data); \ break; \ + case TSDB_DATA_TYPE_TIMESTAMP:\ case TSDB_DATA_TYPE_BIGINT: \ (_v) = (_finalType)(GET_INT64_VAL(_data)); \ break; \ @@ -44,6 +67,43 @@ extern "C" { } \ } while (0) +#define SET_TYPED_DATA(_v, _type, _data) \ + do { \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ + *(int8_t *)(_v) = (int8_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + *(uint8_t *)(_v) = (uint8_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + *(int16_t *)(_v) = (int16_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + *(uint16_t *)(_v) = (uint16_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_BIGINT: \ + *(int64_t *)(_v) = (int64_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + *(uint64_t *)(_v) = (uint64_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + *(float *)(_v) = (float)(_data); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + *(double *)(_v) = (double)(_data); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + *(uint32_t *)(_v) = (uint32_t)(_data); \ + break; \ + default: \ + *(int32_t *)(_v) = (int32_t)(_data); \ + break; \ + } \ + } while (0) + #define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) #define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) @@ -59,6 +119,70 @@ extern "C" { #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) #define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) +static FORCE_INLINE bool isNull(const char *val, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; + case TSDB_DATA_TYPE_TINYINT: + return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; + case TSDB_DATA_TYPE_SMALLINT: + return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; + case TSDB_DATA_TYPE_INT: + return *(uint32_t *)val == TSDB_DATA_INT_NULL; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; + case TSDB_DATA_TYPE_FLOAT: + return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; + case TSDB_DATA_TYPE_DOUBLE: + return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; + case TSDB_DATA_TYPE_NCHAR: + return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; + case TSDB_DATA_TYPE_BINARY: + return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; + case TSDB_DATA_TYPE_UTINYINT: + return *(uint8_t*) val == TSDB_DATA_UTINYINT_NULL; + case TSDB_DATA_TYPE_USMALLINT: + return *(uint16_t*) val == TSDB_DATA_USMALLINT_NULL; + case TSDB_DATA_TYPE_UINT: + return *(uint32_t*) val == TSDB_DATA_UINT_NULL; + case TSDB_DATA_TYPE_UBIGINT: + return *(uint64_t*) val == TSDB_DATA_UBIGINT_NULL; + + default: + return false; + }; +} + +typedef struct tDataTypeDescriptor { + int16_t type; + int16_t nameLen; + int32_t bytes; + char * name; + int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize); + int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize); + void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minindex, int16_t *maxindex, int16_t *numofnull); +} tDataTypeDescriptor; + +extern tDataTypeDescriptor tDataTypes[15]; + +bool isValidDataType(int32_t type); + +void setVardataNull(char* val, int32_t type); +void setNull(char *val, int32_t type, int32_t bytes); +void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); +void *getNullValue(int32_t type); + +void assignVal(char *val, const char *src, int32_t len, int32_t type); +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); + +int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned); + +#define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) + #ifdef __cplusplus } #endif diff --git a/src/inc/twal.h b/src/inc/twal.h index 1645de77aacf020048dc21cd47907163089f269f..bce398d6f95518379c1cd4c0d95a4f6324aab4d7 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -65,6 +65,7 @@ void walFsync(twalh, bool forceFsync); int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); uint64_t walGetVersion(twalh); +void walResetVersion(twalh, uint64_t newVer); #ifdef __cplusplus } diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 7c9ebd8a0b9d96090134bd23c2061c857dfcd5f1..39bd2f46c3e41927389ab4333e27a1cee717aaed 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -60,6 +60,7 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId); int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); +int32_t vnodeSync(int32_t vgId); int32_t vnodeClose(int32_t vgId); // vnodeMgmt @@ -78,7 +79,7 @@ void vnodeFreeFromWQueue(void *pVnode, SVWriteMsg *pWrite); int32_t vnodeProcessWrite(void *pVnode, void *pHead, int32_t qtype, void *pRspRet); // vnodeSync -void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code); +void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force); // vnodeRead int32_t vnodeWriteToRQueue(void *pVnode, void *pCont, int32_t contLen, int8_t qtype, void *rparam); @@ -89,4 +90,4 @@ int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/kit/CMakeLists.txt b/src/kit/CMakeLists.txt index bf77d856f9f772aeffb42f7f85d51a5841943076..66e8cf73988ab25db7544b9a52215d2279630c63 100644 --- a/src/kit/CMakeLists.txt +++ b/src/kit/CMakeLists.txt @@ -1,7 +1,6 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(shell) ADD_SUBDIRECTORY(taosdemo) -ADD_SUBDIRECTORY(taosdemox) ADD_SUBDIRECTORY(taosdump) diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 45da99e5722968591ffccf16432a8cb84d3557ff..d36c1e3fccc4ee7c5eae359f975e2ac1faa6c135 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) @@ -9,14 +9,14 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(./src SRC) LIST(REMOVE_ITEM SRC ./src/shellWindows.c) LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) - ADD_EXECUTABLE(shell ${SRC}) - + ADD_EXECUTABLE(shell ${SRC}) + IF (TD_SOMODE_STATIC) TARGET_LINK_LIBRARIES(shell taos_static) ELSE () TARGET_LINK_LIBRARIES(shell taos) ENDIF () - + SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ELSEIF (TD_WINDOWS) LIST(APPEND SRC ./src/shellEngine.c) @@ -27,7 +27,7 @@ ELSEIF (TD_WINDOWS) IF (TD_POWER) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME power) - ELSE () + ELSE () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () ELSEIF (TD_DARWIN) @@ -36,8 +36,12 @@ ELSEIF (TD_DARWIN) LIST(APPEND SRC ./src/shellDarwin.c) LIST(APPEND SRC ./src/shellCommand.c) LIST(APPEND SRC ./src/shellImport.c) + LIST(APPEND SRC ./src/shellCheck.c) ADD_EXECUTABLE(shell ${SRC}) - TARGET_LINK_LIBRARIES(shell taos_static) + # linking with dylib + TARGET_LINK_LIBRARIES(shell taos) + # linking taos statically + # TARGET_LINK_LIBRARIES(shell taos_static) SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ENDIF () diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index 24156956174364ae1be6abfa2af4c3db7c0db712..d0b7149541eb5ec2b9405e4989e35baa1372f16c 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -30,6 +30,8 @@ #define MAX_COMMAND_SIZE 65536 #define HISTORY_FILE ".taos_history" +#define DEFAULT_RES_SHOW_NUM 100 + typedef struct SShellHistory { char* hist[MAX_HISTORY_SIZE]; int hstart; @@ -49,6 +51,7 @@ typedef struct SShellArguments { char file[TSDB_FILENAME_LEN]; char dir[TSDB_FILENAME_LEN]; int threadNum; + int check; char* commands; int abort; int port; @@ -62,14 +65,16 @@ extern TAOS* shellInit(SShellArguments* args); extern void* shellLoopQuery(void* arg); extern void taos_error(TAOS_RES* tres, int64_t st); extern int regex_match(const char* s, const char* reg, int cflags); -void shellReadCommand(TAOS* con, char command[]); +int32_t shellReadCommand(TAOS* con, char command[]); int32_t shellRunCommand(TAOS* con, char* command); void shellRunCommandOnServer(TAOS* con, char command[]); void read_history(); void write_history(); void source_file(TAOS* con, char* fptr); void source_dir(TAOS* con, SShellArguments* args); +void shellCheck(TAOS* con, SShellArguments* args); void get_history_path(char* history); +void shellCheck(TAOS* con, SShellArguments* args); void cleanup_handler(void* arg); void exitShell(); int shellDumpResult(TAOS_RES* con, char* fname, int* error_no, bool printMode); diff --git a/src/kit/shell/src/shellCheck.c b/src/kit/shell/src/shellCheck.c new file mode 100644 index 0000000000000000000000000000000000000000..b88244ea018291fbdc98165a8665949f618e3291 --- /dev/null +++ b/src/kit/shell/src/shellCheck.c @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _GNU_SOURCE +#define _XOPEN_SOURCE +#define _DEFAULT_SOURCE + +#include "os.h" +#include "shell.h" +#include "shellCommand.h" +#include "tglobal.h" +#include "tutil.h" + +#define SHELL_SQL_LEN 1024 +static int32_t tbNum = 0; +static int32_t tbMallocNum = 0; +static char ** tbNames = NULL; +static int32_t checkedNum = 0; +static int32_t errorNum = 0; + +typedef struct { + pthread_t threadID; + int threadIndex; + int totalThreads; + void * taos; + char * db; +} ShellThreadObj; + +static int32_t shellUseDb(TAOS *con, char *db) { + if (db == NULL) { + fprintf(stdout, "no dbname input\n"); + return -1; + } + + char sql[SHELL_SQL_LEN] = {0}; + snprintf(sql, SHELL_SQL_LEN, "use %s", db); + + TAOS_RES *pSql = taos_query(con, sql); + int32_t code = taos_errno(pSql); + if (code != 0) { + fprintf(stdout, "failed to execute sql:%s since %s", sql, taos_errstr(pSql)); + } + + taos_free_result(pSql); + return code; +} + +static int32_t shellShowTables(TAOS *con, char *db) { + char sql[SHELL_SQL_LEN] = {0}; + snprintf(sql, SHELL_SQL_LEN, "show %s.tables", db); + + TAOS_RES *pSql = taos_query(con, sql); + int32_t code = taos_errno(pSql); + + if (code != 0) { + fprintf(stdout, "failed to execute sql:%s since %s\n", sql, taos_errstr(pSql)); + } else { + TAOS_ROW row; + while ((row = taos_fetch_row(pSql))) { + int32_t tbIndex = tbNum++; + if (tbMallocNum < tbNum) { + tbMallocNum = (tbMallocNum * 2 + 1); + tbNames = realloc(tbNames, tbMallocNum * sizeof(char *)); + if (tbNames == NULL) { + fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum); + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + break; + } + } + + tbNames[tbIndex] = malloc(TSDB_TABLE_NAME_LEN); + strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN); + if (tbIndex % 100000 == 0 && tbIndex != 0) { + fprintf(stdout, "%d tablenames fetched\n", tbIndex); + } + } + } + + taos_free_result(pSql); + + fprintf(stdout, "total %d tablenames fetched, over\n", tbNum); + return code; +} + +static void shellFreeTbnames() { + for (int32_t i = 0; i < tbNum; ++i) { + free(tbNames[i]); + } + free(tbNames); +} + +static void *shellCheckThreadFp(void *arg) { + ShellThreadObj *pThread = (ShellThreadObj *)arg; + + int32_t interval = tbNum / pThread->totalThreads + 1; + int32_t start = pThread->threadIndex * interval; + int32_t end = (pThread->threadIndex + 1) * interval; + + if (end > tbNum) end = tbNum + 1; + + char file[32] = {0}; + snprintf(file, 32, "tb%d.txt", pThread->threadIndex); + + FILE *fp = fopen(file, "w"); + if (!fp) { + fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno)); + return NULL; + } + + char sql[SHELL_SQL_LEN]; + for (int32_t t = start; t < end; ++t) { + char *tbname = tbNames[t]; + if (tbname == NULL) break; + + snprintf(sql, SHELL_SQL_LEN, "select * from %s limit 1", tbname); + + TAOS_RES *pSql = taos_query(pThread->taos, sql); + int32_t code = taos_errno(pSql); + if (code != 0) { + int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); + fwrite(sql, 1, len, fp); + atomic_add_fetch_32(&errorNum, 1); + } + + int32_t cnum = atomic_add_fetch_32(&checkedNum, 1); + if (cnum % 5000 == 0 && cnum != 0) { + fprintf(stdout, "%d tables checked\n", cnum); + } + + taos_free_result(pSql); + } + + fsync(fileno(fp)); + fclose(fp); + + return NULL; +} + +static void shellRunCheckThreads(TAOS *con, SShellArguments *args) { + pthread_attr_t thattr; + ShellThreadObj *threadObj = (ShellThreadObj *)calloc(args->threadNum, sizeof(ShellThreadObj)); + for (int t = 0; t < args->threadNum; ++t) { + ShellThreadObj *pThread = threadObj + t; + pThread->threadIndex = t; + pThread->totalThreads = args->threadNum; + pThread->taos = con; + pThread->db = args->database; + + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { + fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); + exit(0); + } + } + + for (int t = 0; t < args->threadNum; ++t) { + pthread_join(threadObj[t].threadID, NULL); + } + + for (int t = 0; t < args->threadNum; ++t) { + taos_close(threadObj[t].taos); + } + free(threadObj); +} + +void shellCheck(TAOS *con, SShellArguments *args) { + int64_t start = taosGetTimestampMs(); + + if (shellUseDb(con, args->database) != 0) { + shellFreeTbnames(); + return; + } + + if (shellShowTables(con, args->database) != 0) { + shellFreeTbnames(); + return; + } + + fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, args->threadNum); + shellRunCheckThreads(con, args); + + int64_t end = taosGetTimestampMs(); + fprintf(stdout, "total %d tables checked, failed:%d, time spent %.2f seconds\n", checkedNum, errorNum, + (end - start) / 1000.0); +} diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index ddf7b21bef354817f67ccda4106abf1bc648ad6b..31ad7046e9176221e10c79b3f2367ea464529438 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -21,6 +21,8 @@ #include "shellCommand.h" #include "tkey.h" +#include "tscLog.h" + #define OPT_ABORT 1 /* �Cabort */ int indicator = 1; @@ -178,7 +180,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } -void shellReadCommand(TAOS *con, char *command) { +int32_t shellReadCommand(TAOS *con, char *command) { unsigned hist_counter = history.hend; char utf8_array[10] = "\0"; Command cmd; @@ -231,7 +233,7 @@ void shellReadCommand(TAOS *con, char *command) { sprintf(command, "%s%s", cmd.buffer, cmd.command); tfree(cmd.buffer); tfree(cmd.command); - return; + return 0; } else { updateBuffer(&cmd); } @@ -322,6 +324,8 @@ void shellReadCommand(TAOS *con, char *command) { insertChar(&cmd, &c, 1); } } + + return 0; } void *shellLoopQuery(void *arg) { @@ -340,14 +344,22 @@ void *shellLoopQuery(void *arg) { return NULL; } + int32_t err = 0; + do { // Read command from shell. memset(command, 0, MAX_COMMAND_SIZE); set_terminal_mode(); - shellReadCommand(con, command); + err = shellReadCommand(con, command); + if (err) { + break; + } reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); + tfree(command); + exitShell(); + pthread_cleanup_pop(1); return NULL; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index a986f2d3cb038557904c7b1309225491b0a57ea1..0eb1248fad8fe78857b255d8861ea825ab501933 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -76,7 +76,11 @@ TAOS *shellInit(SShellArguments *args) { args->user = TSDB_DEFAULT_USER; } - taos_init(); + if (taos_init()) { + printf("failed to init taos\n"); + fflush(stdout); + return NULL; + } // Connect to the database. TAOS *con = NULL; @@ -117,12 +121,17 @@ TAOS *shellInit(SShellArguments *args) { taos_close(con); exit(EXIT_SUCCESS); } + + if (args->check != 0) { + shellCheck(con, args); + taos_close(con); + exit(EXIT_SUCCESS); + } #endif return con; } - static bool isEmptyCommand(const char* cmd) { for (char c = *cmd++; c != 0; c = *cmd++) { if (c != ' ' && c != '\t' && c != ';') { @@ -320,7 +329,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands - int error_no = 0; + int error_no = 0; + int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { atomic_store_64(&result, 0); @@ -337,7 +347,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } else { int num_rows_affacted = taos_affected_rows(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d row(s) affected (%.6fs)\n", num_rows_affacted, (et - st) / 1E6); + printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); } printf("\n"); @@ -387,10 +397,13 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } time_t tt; + int32_t ms = 0; if (precision == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)(val / 1000000); + ms = val % 1000000; } else { tt = (time_t)(val / 1000); + ms = val % 1000; } /* comment out as it make testcases like select_with_tags.sim fail. @@ -404,14 +417,22 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { #ifdef WINDOWS if (tt < 0) tt = 0; #endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } struct tm* ptm = localtime(&tt); size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm); if (precision == TSDB_TIME_PRECISION_MICRO) { - sprintf(buf + pos, ".%06d", (int)(val % 1000000)); + sprintf(buf + pos, ".%06d", ms); } else { - sprintf(buf + pos, ".%03d", (int)(val % 1000)); + sprintf(buf + pos, ".%03d", ms); } return buf; @@ -470,7 +491,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { wordexp_t full_path; - if (wordexp(fname, &full_path, 0) != 0) { + if (wordexp((char *)fname, &full_path, 0) != 0) { fprintf(stderr, "ERROR: invalid file name: %s\n", fname); return -1; } @@ -644,6 +665,17 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le } +bool isSelectQuery(TAOS_RES* tres) { + char *sql = tscGetSqlStr(tres); + + if (regex_match(sql, "^[\t ]*select[ \t]*", REG_EXTENDED | REG_ICASE)) { + return true; + } + + return false; +} + + static int verticalPrintResult(TAOS_RES* tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { @@ -662,18 +694,33 @@ static int verticalPrintResult(TAOS_RES* tres) { } } + uint64_t resShowMaxNum = UINT64_MAX; + + if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) { + resShowMaxNum = DEFAULT_RES_SHOW_NUM; + } + int numOfRows = 0; - do { - printf("*************************** %d.row ***************************\n", numOfRows + 1); - int32_t* length = taos_fetch_lengths(tres); - for (int i = 0; i < num_fields; i++) { - TAOS_FIELD* field = fields + i; + int showMore = 1; + do { + if (numOfRows < resShowMaxNum) { + printf("*************************** %d.row ***************************\n", numOfRows + 1); - int padding = (int)(maxColNameLen - strlen(field->name)); - printf("%*.s%s: ", padding, " ", field->name); + int32_t* length = taos_fetch_lengths(tres); - printField((const char*)row[i], field, 0, length[i], precision); - putchar('\n'); + for (int i = 0; i < num_fields; i++) { + TAOS_FIELD* field = fields + i; + + int padding = (int)(maxColNameLen - strlen(field->name)); + printf("%*.s%s: ", padding, " ", field->name); + + printField((const char*)row[i], field, 0, length[i], precision); + putchar('\n'); + } + } else if (showMore) { + printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); + printf("[You can add limit statement to get more or redirect results to specific file to get all.]\n"); + showMore = 0; } numOfRows++; @@ -780,16 +827,31 @@ static int horizontalPrintResult(TAOS_RES* tres) { printHeader(fields, width, num_fields); + uint64_t resShowMaxNum = UINT64_MAX; + + if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) && !tscIsQueryWithLimit(tres)) { + resShowMaxNum = DEFAULT_RES_SHOW_NUM; + } + int numOfRows = 0; + int showMore = 1; + do { int32_t* length = taos_fetch_lengths(tres); - for (int i = 0; i < num_fields; i++) { - putchar(' '); - printField((const char*)row[i], fields + i, width[i], length[i], precision); - putchar(' '); - putchar('|'); + if (numOfRows < resShowMaxNum) { + for (int i = 0; i < num_fields; i++) { + putchar(' '); + printField((const char*)row[i], fields + i, width[i], length[i], precision); + putchar(' '); + putchar('|'); + } + putchar('\n'); + } else if (showMore) { + printf("[100 Rows showed, and more rows are fetching but will not be showed. You can ctrl+c to stop or wait.]\n"); + printf("[You can add limit statement to show more or redirect results to specific file to get all.]\n"); + showMore = 0; } - putchar('\n'); + numOfRows++; row = taos_fetch_row(tres); } while(row != NULL); diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 9eb30ccdccc60d9b975236036b7f3aab0675b196..3f6b3da9bf4b1f80f58a06613e8571ca16892c46 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -45,9 +45,10 @@ static struct argp_option options[] = { {"file", 'f', "FILE", 0, "Script to run without enter the shell."}, {"directory", 'D', "DIRECTORY", 0, "Use multi-thread to import all SQL files in the directory separately."}, {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, + {"check", 'k', "CHECK", 0, "Check tables."}, {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, {"timezone", 't', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup."}, + {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."}, {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, {0}}; @@ -130,6 +131,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { return -1; } break; + case 'k': + arguments->check = atoi(arg); + break; case 'd': arguments->database = arg; break; @@ -172,7 +176,7 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { } } -void shellReadCommand(TAOS *con, char *command) { +int32_t shellReadCommand(TAOS *con, char *command) { unsigned hist_counter = history.hend; char utf8_array[10] = "\0"; Command cmd; @@ -186,6 +190,10 @@ void shellReadCommand(TAOS *con, char *command) { while (1) { c = (char)getchar(); // getchar() return an 'int' value + if (c == EOF) { + return c; + } + if (c < 0) { // For UTF-8 int count = countPrefixOnes(c); utf8_array[0] = c; @@ -225,7 +233,7 @@ void shellReadCommand(TAOS *con, char *command) { sprintf(command, "%s%s", cmd.buffer, cmd.command); tfree(cmd.buffer); tfree(cmd.command); - return; + return 0; } else { updateBuffer(&cmd); } @@ -316,6 +324,8 @@ void shellReadCommand(TAOS *con, char *command) { insertChar(&cmd, &c, 1); } } + + return 0; } void *shellLoopQuery(void *arg) { @@ -333,12 +343,17 @@ void *shellLoopQuery(void *arg) { uError("failed to malloc command"); return NULL; } + + int32_t err = 0; do { // Read command from shell. memset(command, 0, MAX_COMMAND_SIZE); set_terminal_mode(); - shellReadCommand(con, command); + err = shellReadCommand(con, command); + if (err) { + break; + } reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 041ad71ccba47cf2b84984af89d25ff15ccf16ce..4c7e550760cecb7c045cb8c94fc431cb5f91812b 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -21,7 +21,7 @@ pthread_t pid; static tsem_t cancelSem; -void shellQueryInterruptHandler(int signum) { +void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); } @@ -110,7 +110,10 @@ int main(int argc, char* argv[]) { } if (args.netTestRole && args.netTestRole[0] != 0) { - taos_init(); + if (taos_init()) { + printf("Failed to init taos"); + exit(EXIT_FAILURE); + } taosNetTest(args.netTestRole, args.host, args.port, args.pktLen); exit(0); } @@ -130,12 +133,10 @@ int main(int argc, char* argv[]) { pthread_create(&spid, NULL, cancelHandler, NULL); /* Interrupt handler. */ - struct sigaction act; - memset(&act, 0, sizeof(struct sigaction)); - - act.sa_handler = shellQueryInterruptHandler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGINT, &act, NULL); + taosSetSignal(SIGTERM, shellQueryInterruptHandler); + taosSetSignal(SIGINT, shellQueryInterruptHandler); + taosSetSignal(SIGHUP, shellQueryInterruptHandler); + taosSetSignal(SIGABRT, shellQueryInterruptHandler); /* Get grant information */ shellGetGrantInfo(con); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 7cb6c75302a0d2417469c5571c88fc382c6919ea..11c1ac34d8bc6058b33699c4d65d9d1224b88d31 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -19,6 +19,10 @@ extern char configDir[]; +void printVersion() { + printf("version: %s\n", version); +} + void printHelp() { char indent[10] = " "; printf("taos shell is used to test the TDengine database\n"); @@ -48,9 +52,11 @@ void printHelp() { printf("%s%s\n", indent, "-t"); printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); printf("%s%s\n", indent, "-n"); - printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup."); + printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync."); printf("%s%s\n", indent, "-l"); printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes."); + printf("%s%s\n", indent, "-V"); + printf("%s%s%s\n", indent, indent, "Print program version."); exit(EXIT_SUCCESS); } @@ -69,6 +75,9 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { // for password else if (strcmp(argv[i], "-p") == 0) { arguments->is_use_passwd = true; + if (i < argc - 1 && argv[i + 1][0] != '-') { + arguments->password = argv[++i]; + } } // for management port else if (strcmp(argv[i], "-P") == 0) { @@ -145,7 +154,6 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } - // For time zone else if (strcmp(argv[i], "-n") == 0) { if (i < argc - 1) { arguments->netTestRole = argv[++i]; @@ -154,7 +162,6 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } - // For time zone else if (strcmp(argv[i], "-l") == 0) { if (i < argc - 1) { arguments->pktLen = atoi(argv[++i]); @@ -163,10 +170,14 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { exit(EXIT_FAILURE); } } + else if (strcmp(argv[i], "-V") == 0) { + printVersion(); + exit(EXIT_SUCCESS); + } // For temperory command TODO else if (strcmp(argv[i], "--help") == 0) { printHelp(); - exit(EXIT_FAILURE); + exit(EXIT_SUCCESS); } else { fprintf(stderr, "wrong options\n"); printHelp(); @@ -210,7 +221,7 @@ void insertChar(Command *cmd, char c) { cmd->command[cmd->cursorOffset++] = c; } -void shellReadCommand(TAOS *con, char command[]) { +int32_t shellReadCommand(TAOS *con, char command[]) { Command cmd; memset(&cmd, 0, sizeof(cmd)); cmd.buffer = (char *)calloc(1, MAX_COMMAND_SIZE); @@ -230,7 +241,7 @@ void shellReadCommand(TAOS *con, char command[]) { cmd.buffer = NULL; free(cmd.command); cmd.command = NULL; - return; + return 0; } else { shellPrintContinuePrompt(); updateBuffer(&cmd); @@ -240,6 +251,8 @@ void shellReadCommand(TAOS *con, char command[]) { insertChar(&cmd, c); } } + + return 0; } void *shellLoopQuery(void *arg) { @@ -247,12 +260,17 @@ void *shellLoopQuery(void *arg) { char *command = malloc(MAX_COMMAND_SIZE); if (command == NULL) return NULL; + int32_t err = 0; + do { memset(command, 0, MAX_COMMAND_SIZE); shellPrintPrompt(); // Read command from shell. - shellReadCommand(con, command); + err = shellReadCommand(con, command); + if (err) { + break; + } } while (shellRunCommand(con, command) == 0); return NULL; diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 91c743939c26ce9bb20e18dd1f701f6eb75fd573..ba27044a8793fec53adc3b85df77443cbf18c600 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -1,20 +1,84 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(inc) + +FIND_PACKAGE(Git) +IF (GIT_FOUND) + MESSAGE("Git found") + EXECUTE_PROCESS( + COMMAND ${GIT_EXECUTABLE} log --pretty=oneline -n 1 ${CMAKE_CURRENT_LIST_DIR}/taosdemo.c + RESULT_VARIABLE RESULT + OUTPUT_VARIABLE TAOSDEMO_COMMIT) + EXECUTE_PROCESS( + COMMAND bash "-c" "echo '${TAOSDEMO_COMMIT}' | awk '{print $1}' | cut -c -9" + RESULT_VARIABLE RESULT + OUTPUT_VARIABLE TAOSDEMO_COMMIT_SHA1) + EXECUTE_PROCESS( + COMMAND ${GIT_EXECUTABLE} status -z -s ${CMAKE_CURRENT_LIST_DIR}/taosdemo.c + RESULT_VARIABLE RESULT + OUTPUT_VARIABLE TAOSDEMO_STATUS) + EXECUTE_PROCESS( + COMMAND bash "-c" "echo '${TAOSDEMO_STATUS}' | awk '{print $1}'" + RESULT_VARIABLE RESULT + OUTPUT_VARIABLE TAOSDEMO_STATUS) + MESSAGE("taosdemo.c status: " ${TAOSDEMO_STATUS}) +ELSE() + MESSAGE("Git not found") + SET(TAOSDEMO_COMMIT_SHA1 "unknown") + SET(TAOSDEMO_STATUS "unknown") +ENDIF (GIT_FOUND) + +STRING(STRIP ${TAOSDEMO_COMMIT_SHA1} TAOSDEMO_COMMIT_SHA1) +MESSAGE("taosdemo's latest commit in short is:" ${TAOSDEMO_COMMIT_SHA1}) +STRING(STRIP ${TAOSDEMO_STATUS} TAOSDEMO_STATUS) + +IF (TAOSDEMO_STATUS MATCHES "M") + SET(TAOSDEMO_STATUS "modified") +ELSE() + SET(TAOSDEMO_STATUS "") +ENDIF () +MESSAGE("taosdemo's status is:" ${TAOSDEMO_STATUS}) + +ADD_DEFINITIONS(-DTAOSDEMO_COMMIT_SHA1="${TAOSDEMO_COMMIT_SHA1}") +ADD_DEFINITIONS(-DTAOSDEMO_STATUS="${TAOSDEMO_STATUS}") + +MESSAGE("VERNUMBER is:" ${VERNUMBER}) +IF ("${VERNUMBER}" STREQUAL "") + SET(TD_VERSION_NUMBER "TDengine-version-unknown") +ELSE() + SET(TD_VERSION_NUMBER ${VERNUMBER}) +ENDIF () +MESSAGE("TD_VERSION_NUMBER is:" ${TD_VERSION_NUMBER}) +ADD_DEFINITIONS(-DTD_VERNUMBER="${TD_VERSION_NUMBER}") IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) - + IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosdemo taos_static) + TARGET_LINK_LIBRARIES(taosdemo taos_static cJson) ELSE () - TARGET_LINK_LIBRARIES(taosdemo taos) + TARGET_LINK_LIBRARIES(taosdemo taos cJson) ENDIF () ELSEIF (TD_WINDOWS) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) - TARGET_LINK_LIBRARIES(taosdemo taos_static) + SET_SOURCE_FILES_PROPERTIES(./taosdemo.c PROPERTIES COMPILE_FLAGS -w) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemo taos_static cJson) + ELSE () + TARGET_LINK_LIBRARIES(taosdemo taos cJson}) + ENDIF () +ELSEIF (TD_DARWIN) + # missing a few dependencies, such as + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(taosdemo ${SRC}) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemo taos_static cJson) + ELSE () + TARGET_LINK_LIBRARIES(taosdemo taos cJson) + ENDIF () ENDIF () + diff --git a/src/kit/taosdemo/insert-interlace.json b/src/kit/taosdemo/insert-interlace.json new file mode 100644 index 0000000000000000000000000000000000000000..344db4fd00fb1fd6616915f7ed1e71b5ee7368d8 --- /dev/null +++ b/src/kit/taosdemo/insert-interlace.json @@ -0,0 +1,57 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 1000, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 20, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}], + "tags": [{"type": "TINYINT", "count":1}] + }] + }] +} diff --git a/src/kit/taosdemo/insert.json b/src/kit/taosdemo/insert.json new file mode 100644 index 0000000000000000000000000000000000000000..f0e3ab1d50dd6cd268568fa7ed81c6ff0a2351c2 --- /dev/null +++ b/src/kit/taosdemo/insert.json @@ -0,0 +1,57 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 10000, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/src/kit/taosdemo/query.json b/src/kit/taosdemo/query.json new file mode 100644 index 0000000000000000000000000000000000000000..33ac120bdae283f64c4419d2885a93d36a7c8093 --- /dev/null +++ b/src/kit/taosdemo/query.json @@ -0,0 +1,19 @@ +{ + "filetype":"query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "yes", + "databases": "dbx", + "specified_table_query": + {"query_interval":1, "concurrent":4, + "sqls": [{"sql": "select last_row(*) from stb where color='red'", "result": "./query_res0.txt"}, + {"sql": "select count(*) from stb_01", "result": "./query_res1.txt"}] + }, + "super_table_query": + {"stblname": "stb", "query_interval":1, "threads":4, + "sqls": [{"sql": "select last_row(*) from xxxx", "result": "./query_res2.txt"}] + } +} diff --git a/src/kit/taosdemox/subscribe.json b/src/kit/taosdemo/subscribe.json similarity index 96% rename from src/kit/taosdemox/subscribe.json rename to src/kit/taosdemo/subscribe.json index f70b1213a884af7d593b3d7366268ce03de1d239..fd33a2e2e2515ac268764c0a7f3f8356e998becd 100644 --- a/src/kit/taosdemox/subscribe.json +++ b/src/kit/taosdemo/subscribe.json @@ -5,7 +5,7 @@ "port": 6030, "user": "root", "password": "taosdata", - "databases": "db", + "databases": "dbx", "specified_table_query": {"concurrent":1, "mode":"sync", "interval":5000, "restart":"yes", "keepProgress":"yes", "sqls": [{"sql": "select avg(col1) from stb01 where col1 > 1;", "result": "./subscribe_res0.txt"}] diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 1d77a6bb6395d7b171a3d79ee75df27ec1826f34..7eb55acda829e0bd21b798a4effd66983e1dcc61 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -13,12 +13,16 @@ * along with this program. If not, see . */ + +/* + when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread. +*/ + #define _GNU_SOURCE +#define CURL_STATICLIB #ifdef LINUX - #include "os.h" #include - #include #include #ifndef _ALPINE #include @@ -27,36 +31,157 @@ #include #include #include - #include #include #include #include #include #include #include -#else - #include +#else #include #include - #include "os.h" -#endif +#endif +#include +#include +#include "cJSON.h" + +#include "os.h" #include "taos.h" +#include "taoserror.h" #include "tutil.h" +#define REQ_EXTRA_BUF_LEN 1024 +#define RESP_BUF_LEN 4096 + extern char configDir[]; -#define BUFFER_SIZE 65536 -#define MAX_DB_NAME_SIZE 64 -#define MAX_TB_NAME_SIZE 64 -#define MAX_DATA_SIZE 16000 -#define MAX_NUM_DATATYPE 10 -#define OPT_ABORT 1 /* –abort */ -#define STRING_LEN 60000 -#define MAX_PREPARED_RAND 1000000 +#define INSERT_JSON_NAME "insert.json" +#define QUERY_JSON_NAME "query.json" +#define SUBSCRIBE_JSON_NAME "subscribe.json" + +enum TEST_MODE { + INSERT_TEST, // 0 + QUERY_TEST, // 1 + SUBSCRIBE_TEST, // 2 + INVAID_TEST +}; + +#define MAX_SQL_SIZE 65536 +#define BUFFER_SIZE (65536*2) +#define MAX_USERNAME_SIZE 64 +#define MAX_PASSWORD_SIZE 64 +#define MAX_DB_NAME_SIZE 64 +#define MAX_HOSTNAME_SIZE 64 +#define MAX_TB_NAME_SIZE 64 +#define MAX_DATA_SIZE 16000 +#define MAX_NUM_DATATYPE 10 +#define OPT_ABORT 1 /* –abort */ +#define STRING_LEN 60000 +#define MAX_PREPARED_RAND 1000000 +#define MAX_FILE_NAME_LEN 256 + +#define MAX_SAMPLES_ONCE_FROM_FILE 10000 +#define MAX_NUM_DATATYPE 10 + +#define MAX_DB_COUNT 8 +#define MAX_SUPER_TABLE_COUNT 200 +#define MAX_COLUMN_COUNT 1024 +#define MAX_TAG_COUNT 128 + +#define MAX_QUERY_SQL_COUNT 100 +#define MAX_QUERY_SQL_LENGTH 256 + +#define MAX_DATABASE_COUNT 256 +#define INPUT_BUF_LEN 256 + +#define DEFAULT_TIMESTAMP_STEP 1 + + +typedef enum CREATE_SUB_TALBE_MOD_EN { + PRE_CREATE_SUBTBL, + AUTO_CREATE_SUBTBL, + NO_CREATE_SUBTBL +} CREATE_SUB_TALBE_MOD_EN; + +typedef enum TALBE_EXISTS_EN { + TBL_NO_EXISTS, + TBL_ALREADY_EXISTS, + TBL_EXISTS_BUTT +} TALBE_EXISTS_EN; + +enum MODE { + SYNC, + ASYNC, + MODE_BUT +}; + +typedef enum enum_INSERT_MODE { + PROGRESSIVE_INSERT_MODE, + INTERLACE_INSERT_MODE, + INVALID_INSERT_MODE +} INSERT_MODE; + +typedef enum enumQUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT +} QUERY_TYPE; + +enum _show_db_index { + TSDB_SHOW_DB_NAME_INDEX, + TSDB_SHOW_DB_CREATED_TIME_INDEX, + TSDB_SHOW_DB_NTABLES_INDEX, + TSDB_SHOW_DB_VGROUPS_INDEX, + TSDB_SHOW_DB_REPLICA_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_DAYS_INDEX, + TSDB_SHOW_DB_KEEP_INDEX, + TSDB_SHOW_DB_CACHE_INDEX, + TSDB_SHOW_DB_BLOCKS_INDEX, + TSDB_SHOW_DB_MINROWS_INDEX, + TSDB_SHOW_DB_MAXROWS_INDEX, + TSDB_SHOW_DB_WALLEVEL_INDEX, + TSDB_SHOW_DB_FSYNC_INDEX, + TSDB_SHOW_DB_COMP_INDEX, + TSDB_SHOW_DB_CACHELAST_INDEX, + TSDB_SHOW_DB_PRECISION_INDEX, + TSDB_SHOW_DB_UPDATE_INDEX, + TSDB_SHOW_DB_STATUS_INDEX, + TSDB_MAX_SHOW_DB +}; + +// -----------------------------------------SHOW TABLES CONFIGURE ------------------------------------- +enum _show_stables_index { + TSDB_SHOW_STABLES_NAME_INDEX, + TSDB_SHOW_STABLES_CREATED_TIME_INDEX, + TSDB_SHOW_STABLES_COLUMNS_INDEX, + TSDB_SHOW_STABLES_METRIC_INDEX, + TSDB_SHOW_STABLES_UID_INDEX, + TSDB_SHOW_STABLES_TID_INDEX, + TSDB_SHOW_STABLES_VGID_INDEX, + TSDB_MAX_SHOW_STABLES +}; + +enum _describe_table_index { + TSDB_DESCRIBE_METRIC_FIELD_INDEX, + TSDB_DESCRIBE_METRIC_TYPE_INDEX, + TSDB_DESCRIBE_METRIC_LENGTH_INDEX, + TSDB_DESCRIBE_METRIC_NOTE_INDEX, + TSDB_MAX_DESCRIBE_METRIC +}; + +typedef struct { + char field[TSDB_COL_NAME_LEN + 1]; + char type[16]; + int length; + char note[128]; +} SColDes; /* Used by main to communicate with parse_opt. */ -typedef struct DemoArguments { +typedef struct SArguments_S { + char * metaFile; + int test_mode; char * host; uint16_t port; char * user; @@ -66,997 +191,5006 @@ typedef struct DemoArguments { char * tb_prefix; char * sqlFile; bool use_metric; + bool drop_database; bool insert_only; + bool answer_yes; + bool debug_print; + bool verbose_print; + bool performance_print; char * output_file; int mode; char * datatype[MAX_NUM_DATATYPE + 1]; int len_of_binary; int num_of_CPR; int num_of_threads; + int insert_interval; + int query_times; + int interlace_rows; int num_of_RPR; + int max_sql_len; int num_of_tables; int num_of_DPT; int abort; - int order; - int rate; + int disorderRatio; + int disorderRange; int method_of_delete; char ** arg_list; -} SDemoArguments; - -#ifdef LINUX - /* The options we understand. */ - static struct argp_option options[] = { - {0, 'h', "host", 0, "The host to connect to TDengine. Default is localhost.", 0}, - {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 1}, - {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, - #ifdef _TD_POWER_ - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'powerdb'.", 3}, - #else - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, - #endif - {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, - {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, - {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, - {0, 's', "sql file", 0, "The select sql file.", 3}, - {0, 'M', 0, 0, "Use metric flag.", 13}, - {0, 'o', "outputfile", 0, "Direct output to the named file. Default is './output.txt'.", 14}, - {0, 'q', "query_mode", 0, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.", 6}, - {0, 'b', "type_of_cols", 0, "The data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'.", 7}, - {0, 'w', "length_of_binary", 0, "The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8", 8}, - {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 3.", 8}, - {0, 'T', "num_of_threads", 0, "The number of threads. Default is 10.", 9}, - {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 1000.", 10}, - {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 11}, - {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 100000.", 12}, - #ifdef _TD_POWER_ - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/power/'.", 14}, - #else - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 14}, - #endif - {0, 'x', 0, 0, "Insert only flag.", 13}, - {0, 'O', "order", 0, "Insert mode--0: In order, 1: Out of order. Default is in order.", 14}, - {0, 'R', "rate", 0, "Out of order data's rate--if order=1 Default 10, min: 0, max: 50.", 14}, - {0, 'D', "delete table", 0, "Delete data methods——0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database", 14}, - {0}}; - - /* Parse a single option. */ - static error_t parse_opt(int key, char *arg, struct argp_state *state) { - /* Get the input argument from argp_parse, which we - know is a pointer to our arguments structure. */ - SDemoArguments *arguments = state->input; - wordexp_t full_path; - char **sptr; - switch (key) { - case 'h': - arguments->host = arg; - break; - case 'p': - arguments->port = atoi(arg); - break; - case 'u': - arguments->user = arg; - break; - case 'P': - arguments->password = arg; - break; - case 'o': - arguments->output_file = arg; - break; - case 's': - arguments->sqlFile = arg; - break; - case 'q': - arguments->mode = atoi(arg); - break; - case 'T': - arguments->num_of_threads = atoi(arg); - break; - case 'r': - arguments->num_of_RPR = atoi(arg); - break; - case 't': - arguments->num_of_tables = atoi(arg); - break; - case 'n': - arguments->num_of_DPT = atoi(arg); - break; - case 'd': - arguments->database = arg; - break; - case 'l': - arguments->num_of_CPR = atoi(arg); - break; - case 'b': - sptr = arguments->datatype; - if (strstr(arg, ",") == NULL) { - if (strcasecmp(arg, "INT") != 0 && strcasecmp(arg, "FLOAT") != 0 && - strcasecmp(arg, "TINYINT") != 0 && strcasecmp(arg, "BOOL") != 0 && - strcasecmp(arg, "SMALLINT") != 0 && - strcasecmp(arg, "BIGINT") != 0 && strcasecmp(arg, "DOUBLE") != 0 && - strcasecmp(arg, "BINARY") && strcasecmp(arg, "NCHAR")) { - argp_error(state, "Invalid data_type!"); - } - sptr[0] = arg; - } else { - int index = 0; - char *dupstr = strdup(arg); - char *running = dupstr; - char *token = strsep(&running, ","); - while (token != NULL) { - if (strcasecmp(token, "INT") != 0 && - strcasecmp(token, "FLOAT") != 0 && - strcasecmp(token, "TINYINT") != 0 && - strcasecmp(token, "BOOL") != 0 && - strcasecmp(token, "SMALLINT") != 0 && - strcasecmp(token, "BIGINT") != 0 && - strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { - argp_error(state, "Invalid data_type!"); - } - sptr[index++] = token; - token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; - } - } - break; - case 'w': - arguments->len_of_binary = atoi(arg); - break; - case 'm': - arguments->tb_prefix = arg; - break; - case 'M': - arguments->use_metric = true; - break; - case 'x': - arguments->insert_only = true; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); - wordfree(&full_path); - break; - case 'O': - arguments->order = atoi(arg); - if (arguments->order > 1 || arguments->order < 0) - { - arguments->order = 0; - } else if (arguments->order == 1) - { - arguments->rate = 10; - } - break; - case 'R': - arguments->rate = atoi(arg); - if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) - { - arguments->rate = 10; - } - break; - case 'a': - arguments->replica = atoi(arg); - if (arguments->replica > 3 || arguments->replica < 1) - { - arguments->replica = 1; - } - break; - case 'D': - arguments->method_of_delete = atoi(arg); - if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) - { - arguments->method_of_delete = 0; - } - break; - case OPT_ABORT: - arguments->abort = 1; - break; - case ARGP_KEY_ARG: - /*arguments->arg_list = &state->argv[state->next-1]; - state->next = state->argc;*/ - argp_usage(state); - break; - - default: - return ARGP_ERR_UNKNOWN; - } - return 0; - } + int64_t totalInsertRows; + int64_t totalAffectedRows; +} SArguments; + +typedef struct SColumn_S { + char field[TSDB_COL_NAME_LEN + 1]; + char dataType[MAX_TB_NAME_SIZE]; + int dataLen; + char note[128]; +} StrColumn; + +typedef struct SSuperTable_S { + char sTblName[MAX_TB_NAME_SIZE+1]; + int childTblCount; + bool childTblExists; // 0: no, 1: yes + int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql + int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table + char childTblPrefix[MAX_TB_NAME_SIZE]; + char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample + char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful + int childTblLimit; + int childTblOffset; + + int multiThreadWriteOneTbl; // 0: no, 1: yes + int interlaceRows; // + int disorderRatio; // 0: no disorder, >0: x% + int disorderRange; // ms or us by database precision + int maxSqlLen; // + + int insertInterval; // insert interval, will override global insert interval + int64_t insertRows; // 0: no limit + int timeStampStep; + char startTimestamp[MAX_TB_NAME_SIZE]; // + char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json + char sampleFile[MAX_FILE_NAME_LEN+1]; + char tagsFile[MAX_FILE_NAME_LEN+1]; + + int columnCount; + StrColumn columns[MAX_COLUMN_COUNT]; + int tagCount; + StrColumn tags[MAX_TAG_COUNT]; + + char* childTblName; + char* colsOfCreateChildTable; + int lenOfOneRow; + int lenOfTagOfOneRow; + + char* sampleDataBuf; + int sampleDataBufSize; + //int sampleRowCount; + //int sampleUsePos; + + int tagSource; // 0: rand, 1: tag sample + char* tagDataBuf; + int tagSampleCount; + int tagUsePos; + + // statistics + int64_t totalInsertRows; + int64_t totalAffectedRows; +} SSuperTable; - static struct argp argp = {options, parse_opt, 0, 0}; - - void parse_args(int argc, char *argv[], SDemoArguments *arguments) { - argp_parse(&argp, argc, argv, 0, 0, arguments); - if (arguments->abort) { - #ifndef _ALPINE - error(10, 0, "ABORTED"); - #else - abort(); - #endif - } - } - -#else - void printHelp() { - char indent[10] = " "; - printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "host, The host to connect to TDengine. Default is localhost."); - printf("%s%s\n", indent, "-p"); - printf("%s%s%s\n", indent, indent, "port, The TCP/IP port number to use for the connection. Default is 0."); - printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "user, The user name to use when connecting to the server. Default is 'root'."); - printf("%s%s\n", indent, "-p"); - #ifdef _TD_POWER_ - printf("%s%s%s\n", indent, indent, "password, The password to use when connecting to the server. Default is 'powerdb'."); - #else - printf("%s%s%s\n", indent, indent, "password, The password to use when connecting to the server. Default is 'taosdata'."); - #endif - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, "database, Destination database. Default is 'test'."); - printf("%s%s\n", indent, "-a"); - printf("%s%s%s\n", indent, indent, "replica, Set the replica parameters of the database, Default 1, min: 1, max: 3."); - printf("%s%s\n", indent, "-m"); - printf("%s%s%s\n", indent, indent, "table_prefix, Table prefix name. Default is 't'."); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s\n", indent, indent, "sql file, The select sql file."); - printf("%s%s\n", indent, "-M"); - printf("%s%s%s\n", indent, indent, "meteric, Use metric flag."); - printf("%s%s\n", indent, "-o"); - printf("%s%s%s\n", indent, indent, "outputfile, Direct output to the named file. Default is './output.txt'."); - printf("%s%s\n", indent, "-q"); - printf("%s%s%s\n", indent, indent, "query_mode, Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); - printf("%s%s\n", indent, "-b"); - printf("%s%s%s\n", indent, indent, "type_of_cols, data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'."); - printf("%s%s\n", indent, "-w"); - printf("%s%s%s\n", indent, indent, "length_of_binary, The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8"); - printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "num_of_cols_per_record, The number of columns per record. Default is 3."); - printf("%s%s\n", indent, "-T"); - printf("%s%s%s\n", indent, indent, "num_of_threads, The number of threads. Default is 10."); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s\n", indent, indent, "num_of_records_per_req, The number of records per request. Default is 1000."); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, "num_of_tables, The number of tables. Default is 10000."); - printf("%s%s\n", indent, "-n"); - printf("%s%s%s\n", indent, indent, "num_of_records_per_table, The number of records per table. Default is 100000."); - printf("%s%s\n", indent, "-c"); - #ifdef _TD_POWER_ - printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/power/'."); - #else - printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); - #endif - printf("%s%s\n", indent, "-x"); - printf("%s%s%s\n", indent, indent, "flag, Insert only flag."); - printf("%s%s\n", indent, "-O"); - printf("%s%s%s\n", indent, indent, "order, Insert mode--0: In order, 1: Out of order. Default is in order."); - printf("%s%s\n", indent, "-R"); - printf("%s%s%s\n", indent, indent, "rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50."); - printf("%s%s\n", indent, "-D"); - printf("%s%s%s\n", indent, indent, "Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database."); - } - - void parse_args(int argc, char *argv[], SDemoArguments *arguments) { - char **sptr; - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-h") == 0) { - arguments->host = argv[++i]; - } else if (strcmp(argv[i], "-p") == 0) { - arguments->port = atoi(argv[++i]); - } else if (strcmp(argv[i], "-u") == 0) { - arguments->user = argv[++i]; - } else if (strcmp(argv[i], "-P") == 0) { - arguments->password = argv[++i]; - } else if (strcmp(argv[i], "-o") == 0) { - arguments->output_file = argv[++i]; - } else if (strcmp(argv[i], "-s") == 0) { - arguments->sqlFile = argv[++i]; - } else if (strcmp(argv[i], "-q") == 0) { - arguments->mode = atoi(argv[++i]); - } else if (strcmp(argv[i], "-T") == 0) { - arguments->num_of_threads = atoi(argv[++i]); - } else if (strcmp(argv[i], "-r") == 0) { - arguments->num_of_RPR = atoi(argv[++i]); - } else if (strcmp(argv[i], "-t") == 0) { - arguments->num_of_tables = atoi(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { - arguments->num_of_DPT = atoi(argv[++i]); - } else if (strcmp(argv[i], "-d") == 0) { - arguments->database = argv[++i]; - } else if (strcmp(argv[i], "-l") == 0) { - arguments->num_of_CPR = atoi(argv[++i]); - } else if (strcmp(argv[i], "-b") == 0) { - sptr = arguments->datatype; - ++i; - if (strstr(argv[i], ",") == NULL) { - if (strcasecmp(argv[i], "INT") != 0 && strcasecmp(argv[i], "FLOAT") != 0 && - strcasecmp(argv[i], "TINYINT") != 0 && strcasecmp(argv[i], "BOOL") != 0 && - strcasecmp(argv[i], "SMALLINT") != 0 && - strcasecmp(argv[i], "BIGINT") != 0 && strcasecmp(argv[i], "DOUBLE") != 0 && - strcasecmp(argv[i], "BINARY") && strcasecmp(argv[i], "NCHAR")) { - fprintf(stderr, "Invalid data_type!\n"); - printHelp(); - exit(EXIT_FAILURE); - } - sptr[0] = argv[i]; - } else { - int index = 0; - char *dupstr = strdup(argv[i]); - char *running = dupstr; - char *token = strsep(&running, ","); - while (token != NULL) { - if (strcasecmp(token, "INT") != 0 && - strcasecmp(token, "FLOAT") != 0 && - strcasecmp(token, "TINYINT") != 0 && - strcasecmp(token, "BOOL") != 0 && - strcasecmp(token, "SMALLINT") != 0 && - strcasecmp(token, "BIGINT") != 0 && - strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { - fprintf(stderr, "Invalid data_type!\n"); - printHelp(); - exit(EXIT_FAILURE); - } - sptr[index++] = token; - token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; - } - } - } else if (strcmp(argv[i], "-w") == 0) { - arguments->len_of_binary = atoi(argv[++i]); - } else if (strcmp(argv[i], "-m") == 0) { - arguments->tb_prefix = argv[++i]; - } else if (strcmp(argv[i], "-M") == 0) { - arguments->use_metric = true; - } else if (strcmp(argv[i], "-x") == 0) { - arguments->insert_only = true; - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "-O") == 0) { - arguments->order = atoi(argv[++i]); - if (arguments->order > 1 || arguments->order < 0) { - arguments->order = 0; - } else if (arguments->order == 1) { - arguments->rate = 10; - } - } else if (strcmp(argv[i], "-R") == 0) { - arguments->rate = atoi(argv[++i]); - if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) { - arguments->rate = 10; - } - } else if (strcmp(argv[i], "-a") == 0) { - arguments->replica = atoi(argv[++i]); - if (arguments->rate > 3 || arguments->rate < 1) { - arguments->rate = 1; - } - } else if (strcmp(argv[i], "-D") == 0) { - arguments->method_of_delete = atoi(argv[++i]); - if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) { - arguments->method_of_delete = 0; - } - } else if (strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(EXIT_FAILURE); - } else { - fprintf(stderr, "wrong options\n"); - printHelp(); - exit(EXIT_FAILURE); - } - } - } - -#endif - -/* ******************************* Structure - * definition******************************* */ -enum MODE { - SYNC, ASYNC -}; typedef struct { + char name[TSDB_DB_NAME_LEN + 1]; + char create_time[32]; + int32_t ntables; + int32_t vgroups; + int16_t replica; + int16_t quorum; + int16_t days; + char keeplist[32]; + int32_t cache; //MB + int32_t blocks; + int32_t minrows; + int32_t maxrows; + int8_t wallevel; + int32_t fsync; + int8_t comp; + int8_t cachelast; + char precision[8]; // time resolution + int8_t update; + char status[16]; +} SDbInfo; + +typedef struct SDbCfg_S { +// int maxtablesPerVnode; + int minRows; + int maxRows; + int comp; + int walLevel; + int cacheLast; + int fsync; + int replica; + int update; + int keep; + int days; + int cache; + int blocks; + int quorum; + char precision[MAX_TB_NAME_SIZE]; +} SDbCfg; + +typedef struct SDataBase_S { + char dbName[MAX_DB_NAME_SIZE]; + bool drop; // 0: use exists, 1: if exists, drop then new create + SDbCfg dbCfg; + int superTblCount; + SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; +} SDataBase; + +typedef struct SDbs_S { + char cfgDir[MAX_FILE_NAME_LEN+1]; + char host[MAX_HOSTNAME_SIZE]; + uint16_t port; + char user[MAX_USERNAME_SIZE]; + char password[MAX_PASSWORD_SIZE]; + char resultFile[MAX_FILE_NAME_LEN+1]; + bool use_metric; + bool insert_only; + bool do_aggreFunc; + bool queryMode; + + int threadCount; + int threadCountByCreateTbl; + int dbCount; + SDataBase db[MAX_DB_COUNT]; + + // statistics + int64_t totalInsertRows; + int64_t totalAffectedRows; + +} SDbs; + +typedef struct SpecifiedQueryInfo_S { + int rate; // 0: unlimit > 0 loop/s + int concurrent; + int sqlCount; + int subscribeMode; // 0: sync, 1: async + int subscribeInterval; // ms + int queryTimes; + int subscribeRestart; + int subscribeKeepProgress; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; +} SpecifiedQueryInfo; + +typedef struct SuperQueryInfo_S { + char sTblName[MAX_TB_NAME_SIZE+1]; + int rate; // 0: unlimit > 0 loop/s + int threadCnt; + int subscribeMode; // 0: sync, 1: async + int subscribeInterval; // ms + int subscribeRestart; + int subscribeKeepProgress; + int queryTimes; + int childTblCount; + char childTblPrefix[MAX_TB_NAME_SIZE]; + int sqlCount; + char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH+1]; + char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN+1]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; + + char* childTblName; +} SuperQueryInfo; + +typedef struct SQueryMetaInfo_S { + char cfgDir[MAX_FILE_NAME_LEN+1]; + char host[MAX_HOSTNAME_SIZE]; + uint16_t port; + char user[MAX_USERNAME_SIZE]; + char password[MAX_PASSWORD_SIZE]; + char dbName[MAX_DB_NAME_SIZE+1]; + char queryMode[MAX_TB_NAME_SIZE]; // taosc, restful + + SpecifiedQueryInfo specifiedQueryInfo; + SuperQueryInfo superQueryInfo; +} SQueryMetaInfo; + +typedef struct SThreadInfo_S { TAOS *taos; int threadID; - char db_name[MAX_DB_NAME_SIZE]; + char db_name[MAX_DB_NAME_SIZE+1]; + uint32_t time_precision; char fp[4096]; - char **datatype; - int len_of_binary; char tb_prefix[MAX_TB_NAME_SIZE]; - int start_table_id; - int end_table_id; - int ncols_per_record; - int nrecords_per_table; - int nrecords_per_request; - int data_of_order; + int start_table_from; + int end_table_to; + int ntables; int data_of_rate; - int64_t start_time; - bool do_aggreFunc; - - char* cols; + uint64_t start_time; + char* cols; bool use_metric; + SSuperTable* superTblInfo; - tsem_t mutex_sem; - int notFinished; + // for async insert tsem_t lock_sem; - int counter; - - // insert delay statitics + int64_t counter; + uint64_t st; + uint64_t et; + int64_t lastTs; + + // sample data + int samplePos; + // statistics + int64_t totalInsertRows; + int64_t totalAffectedRows; + + // insert delay statistics int64_t cntDelay; int64_t totalDelay; int64_t avgDelay; int64_t maxDelay; int64_t minDelay; - -} info; -typedef struct { - TAOS *taos; - - char tb_name[MAX_TB_NAME_SIZE]; - int64_t timestamp; - int target; - int counter; - int nrecords_per_request; - int ncols_per_record; - char **data_type; - int len_of_binary; - int data_of_order; - int data_of_rate; +} threadInfo; - tsem_t *mutex_sem; - int *notFinished; - tsem_t *lock_sem; -} sTable; +#ifdef WINDOWS +#define _CRT_RAND_S -/* ******************************* Global - * variables******************************* */ -char *aggreFunc[] = {"*", "count(*)", "avg(f1)", "sum(f1)", "max(f1)", "min(f1)", "first(f1)", "last(f1)"}; +#include +#include +typedef unsigned __int32 uint32_t; -void queryDB(TAOS *taos, char *command); +#pragma comment ( lib, "ws2_32.lib" ) +// Some old MinGW/CYGWIN distributions don't define this: +#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING + #define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 +#endif // ENABLE_VIRTUAL_TERMINAL_PROCESSING -void *readTable(void *sarg); +static HANDLE g_stdoutHandle; +static DWORD g_consoleMode; -void *readMetric(void *sarg); +static void setupForAnsiEscape(void) { + DWORD mode = 0; + g_stdoutHandle = GetStdHandle(STD_OUTPUT_HANDLE); -void *syncWrite(void *sarg); + if(g_stdoutHandle == INVALID_HANDLE_VALUE) { + exit(GetLastError()); + } -void *deleteTable(); + if(!GetConsoleMode(g_stdoutHandle, &mode)) { + exit(GetLastError()); + } -void *asyncWrite(void *sarg); + g_consoleMode = mode; -int generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary); + // Enable ANSI escape codes + mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; -void rand_string(char *str, int size); + if(!SetConsoleMode(g_stdoutHandle, mode)) { + exit(GetLastError()); + } +} -void init_rand_data(); +static void resetAfterAnsiEscape(void) { + // Reset colors + printf("\x1b[0m"); -double getCurrentTime(); + // Reset console mode + if(!SetConsoleMode(g_stdoutHandle, g_consoleMode)) { + exit(GetLastError()); + } +} -void callBack(void *param, TAOS_RES *res, int code); -void multiThreadCreateTable(char* cols, bool use_metric, int threads, int ntables, char* db_name, char* tb_prefix, char *ip_addr, uint16_t port, char *user, char *pass); -void querySqlFile(TAOS* taos, char* sqlFile); +static int taosRandom() +{ + int number; + rand_s(&number); -int main(int argc, char *argv[]) { - SDemoArguments arguments = { NULL, // host - 0, // port - "root", // user - #ifdef _TD_POWER_ - "powerdb", // password - #else - "taosdata", // password - #endif - "test", // database - 1, // replica - "t", // tb_prefix - NULL, - false, // use_metric - false, // insert_only - "./output.txt", // output_file - 0, // mode - { - "int", // datatype - "int", - "int", - "int", - "int", - "int", - "int", - "float" - }, - 8, // len_of_binary - 1, // num_of_CPR - 1, // num_of_connections/thread - 1, // num_of_RPR - 1, // num_of_tables - 50000, // num_of_DPT - 0, // abort - 0, // order - 0, // rate - 0, // method_of_delete - NULL // arg_list - }; - - /* Parse our arguments; every option seen by parse_opt will be - reflected in arguments. */ - // For demo use, change default values for some parameters; - arguments.num_of_tables = 10000; - arguments.num_of_CPR = 3; - arguments.num_of_threads = 10; - arguments.num_of_DPT = 100000; - arguments.num_of_RPR = 1000; - arguments.use_metric = true; - arguments.insert_only = false; - // end change - - parse_args(argc, argv, &arguments); - - enum MODE query_mode = arguments.mode; - char *ip_addr = arguments.host; - uint16_t port = arguments.port; - char *user = arguments.user; - char *pass = arguments.password; - char *db_name = arguments.database; - char *tb_prefix = arguments.tb_prefix; - int len_of_binary = arguments.len_of_binary; - int ncols_per_record = arguments.num_of_CPR; - int order = arguments.order; - int rate = arguments.rate; - int method_of_delete = arguments.method_of_delete; - int ntables = arguments.num_of_tables; - int threads = arguments.num_of_threads; - int nrecords_per_table = arguments.num_of_DPT; - int nrecords_per_request = arguments.num_of_RPR; - bool use_metric = arguments.use_metric; - bool insert_only = arguments.insert_only; - char **data_type = arguments.datatype; - int count_data_type = 0; - char dataString[STRING_LEN]; - bool do_aggreFunc = true; - int replica = arguments.replica; + return number; +} +#else +static void setupForAnsiEscape(void) {} - if (NULL != arguments.sqlFile) { - TAOS* qtaos = taos_connect(ip_addr, user, pass, db_name, port); - querySqlFile(qtaos, arguments.sqlFile); - taos_close(qtaos); - return 0; - } - init_rand_data(); +static void resetAfterAnsiEscape(void) { + // Reset colors + printf("\x1b[0m"); +} - memset(dataString, 0, STRING_LEN); - int len = 0; +#include - if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0 || strcasecmp(data_type[0], "NCHAR") == 0 ) { - do_aggreFunc = false; - } - for (; count_data_type <= MAX_NUM_DATATYPE; count_data_type++) { - if (data_type[count_data_type] == NULL) { - break; - } +static int taosRandom() +{ + srand(time(NULL)); + return rand(); +} - len += snprintf(dataString + len, STRING_LEN - len, "%s ", data_type[count_data_type]); - } +#endif - FILE *fp = fopen(arguments.output_file, "a"); - if (NULL == fp) { - fprintf(stderr, "Failed to open %s for writing\n", arguments.output_file); - return 1; - }; - - time_t tTime = time(NULL); - struct tm tm = *localtime(&tTime); - printf("###################################################################\n"); - printf("# Server IP: %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port); - printf("# User: %s\n", user); - printf("# Password: %s\n", pass); - printf("# Use metric: %s\n", use_metric ? "true" : "false"); - printf("# Datatype of Columns: %s\n", dataString); - printf("# Binary Length(If applicable): %d\n", - (strncasecmp(dataString, "BINARY", 6) == 0 || strncasecmp(dataString, "NCHAR", 5) == 0) ? len_of_binary : -1); - printf("# Number of Columns per record: %d\n", ncols_per_record); - printf("# Number of Threads: %d\n", threads); - printf("# Number of Tables: %d\n", ntables); - printf("# Number of Data per Table: %d\n", nrecords_per_table); - printf("# Records/Request: %d\n", nrecords_per_request); - printf("# Database name: %s\n", db_name); - printf("# Table prefix: %s\n", tb_prefix); - if (order == 1) - { - printf("# Data order: %d\n", order); - printf("# Data out of order rate: %d\n", rate); - - } - printf("# Delete method: %d\n", method_of_delete); - printf("# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - printf("###################################################################\n\n"); - printf("Press enter key to continue"); - (void)getchar(); - - fprintf(fp, "###################################################################\n"); - fprintf(fp, "# Server IP: %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port); - fprintf(fp, "# User: %s\n", user); - fprintf(fp, "# Password: %s\n", pass); - fprintf(fp, "# Use metric: %s\n", use_metric ? "true" : "false"); - fprintf(fp, "# Datatype of Columns: %s\n", dataString); - fprintf(fp, "# Binary Length(If applicable): %d\n", - (strncasecmp(dataString, "BINARY", 6) == 0 || strncasecmp(dataString, "NCHAR", 5) == 0) ? len_of_binary : -1); - fprintf(fp, "# Number of Columns per record: %d\n", ncols_per_record); - fprintf(fp, "# Number of Threads: %d\n", threads); - fprintf(fp, "# Number of Tables: %d\n", ntables); - fprintf(fp, "# Number of Data per Table: %d\n", nrecords_per_table); - fprintf(fp, "# Records/Request: %d\n", nrecords_per_request); - fprintf(fp, "# Database name: %s\n", db_name); - fprintf(fp, "# Table prefix: %s\n", tb_prefix); - if (order == 1) - { - printf("# Data order: %d\n", order); - printf("# Data out of order rate: %d\n", rate); - - } - fprintf(fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - fprintf(fp, "###################################################################\n\n"); - fprintf(fp, "| WRecords | Records/Second | Requests/Second | WLatency(ms) |\n"); - - taos_init(); - TAOS *taos = taos_connect(ip_addr, user, pass, NULL, port); - if (taos == NULL) { - fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - return 1; - } - char command[BUFFER_SIZE] = "\0"; +static int createDatabasesAndStables(); +static void createChildTables(); +static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); - sprintf(command, "drop database %s;", db_name); - TAOS_RES* res = taos_query(taos, command); - taos_free_result(res); +/* ************ Global variables ************ */ - sprintf(command, "create database %s replica %d;", db_name, replica); - res = taos_query(taos, command); - taos_free_result(res); +int32_t randint[MAX_PREPARED_RAND]; +int64_t randbigint[MAX_PREPARED_RAND]; +float randfloat[MAX_PREPARED_RAND]; +double randdouble[MAX_PREPARED_RAND]; +char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", + "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; + +SArguments g_args = { + NULL, // metaFile + 0, // test_mode + "127.0.0.1", // host + 6030, // port + "root", // user + #ifdef _TD_POWER_ + "powerdb", // password + #else + "taosdata", // password + #endif + "test", // database + 1, // replica + "t", // tb_prefix + NULL, // sqlFile + true, // use_metric + true, // drop_database + true, // insert_only + false, // debug_print + false, // verbose_print + false, // performance statistic print + false, // answer_yes; + "./output.txt", // output_file + 0, // mode : sync or async + { + "TINYINT", // datatype + "SMALLINT", + "INT", + "BIGINT", + "FLOAT", + "DOUBLE", + "BINARY", + "NCHAR", + "BOOL", + "TIMESTAMP" + }, + 16, // len_of_binary + 10, // num_of_CPR + 10, // num_of_connections/thread + 0, // insert_interval + 1, // query_times + 0, // interlace_rows; + 100, // num_of_RPR + TSDB_PAYLOAD_SIZE, // max_sql_len + 10000, // num_of_tables + 10000, // num_of_DPT + 0, // abort + 0, // disorderRatio + 1000, // disorderRange + 1, // method_of_delete + NULL // arg_list +}; - char cols[STRING_LEN] = "\0"; - int colIndex = 0; - len = 0; - for (; colIndex < ncols_per_record - 1; colIndex++) { - if (strcasecmp(data_type[colIndex % count_data_type], "BINARY") != 0 && strcasecmp(data_type[colIndex % count_data_type], "NCHAR") != 0) { - len += snprintf(cols + len, STRING_LEN - len, ",f%d %s", colIndex + 1, data_type[colIndex % count_data_type]); - } else { - len += snprintf(cols + len, STRING_LEN - len, ",f%d %s(%d)", colIndex + 1, data_type[colIndex % count_data_type], len_of_binary); - } - } - if (strcasecmp(data_type[colIndex % count_data_type], "BINARY") != 0 && strcasecmp(data_type[colIndex % count_data_type], "NCHAR") != 0){ - len += snprintf(cols + len, STRING_LEN - len, ",f%d %s", colIndex + 1, data_type[colIndex % count_data_type]); - } else { - len += snprintf(cols + len, STRING_LEN - len, ",f%d %s(%d)", colIndex + 1, data_type[colIndex % count_data_type], len_of_binary); - } +static SDbs g_Dbs; +static int g_totalChildTables = 0; +static SQueryMetaInfo g_queryInfo; +static FILE * g_fpOfInsertResult = NULL; - if (use_metric) { - /* Create metric table */ - printf("Creating meters super table...\n"); - snprintf(command, BUFFER_SIZE, "create table if not exists %s.meters (ts timestamp%s) tags (areaid int, loc binary(10))", db_name, cols); - queryDB(taos, command); - printf("meters created!\n"); - } - taos_close(taos); - - /* Wait for table to create */ - multiThreadCreateTable(cols, use_metric, threads, ntables, db_name, tb_prefix, ip_addr, port, user, pass); - - /* Insert data */ - double ts = getCurrentTime(); - printf("Inserting data......\n"); - pthread_t *pids = malloc(threads * sizeof(pthread_t)); - info *infos = malloc(threads * sizeof(info)); - - memset(pids, 0, threads * sizeof(pthread_t)); - memset(infos, 0, threads * sizeof(info)); +#define debugPrint(fmt, ...) \ + do { if (g_args.debug_print || g_args.verbose_print) \ + fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0) - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } +#define verbosePrint(fmt, ...) \ + do { if (g_args.verbose_print) \ + fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0) - int b = 0; - if (threads != 0) - b = ntables % threads; - int last = 0; - for (int i = 0; i < threads; i++) { - info *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - tstrncpy(t_info->tb_prefix, tb_prefix, MAX_TB_NAME_SIZE); - t_info->datatype = data_type; - t_info->ncols_per_record = ncols_per_record; - t_info->nrecords_per_table = nrecords_per_table; - t_info->start_time = 1500000000000; - t_info->taos = taos_connect(ip_addr, user, pass, db_name, port); - t_info->len_of_binary = len_of_binary; - t_info->nrecords_per_request = nrecords_per_request; - t_info->start_table_id = last; - t_info->data_of_order = order; - t_info->data_of_rate = rate; - t_info->end_table_id = i < b ? last + a : last + a - 1; - last = t_info->end_table_id + 1; - t_info->counter = 0; - t_info->minDelay = INT16_MAX; +#define performancePrint(fmt, ...) \ + do { if (g_args.performance_print) \ + fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0) - tsem_init(&(t_info->mutex_sem), 0, 1); - t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1; - tsem_init(&(t_info->lock_sem), 0, 0); +#define errorPrint(fmt, ...) \ + do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0) - if (query_mode == SYNC) { - pthread_create(pids + i, NULL, syncWrite, t_info); - } else { - pthread_create(pids + i, NULL, asyncWrite, t_info); - } - } - for (int i = 0; i < threads; i++) { - pthread_join(pids[i], NULL); - } - double t = getCurrentTime() - ts; - if (query_mode == SYNC) { - printf("SYNC Insert with %d connections:\n", threads); - } else { - printf("ASYNC Insert with %d connections:\n", threads); - } +/////////////////////////////////////////////////// - fprintf(fp, "|%"PRIu64" | %10.2f | %10.2f | %10.4f |\n\n", - (int64_t)ntables * nrecords_per_table, ntables * nrecords_per_table / t, - ((int64_t)ntables * nrecords_per_table) / (t * nrecords_per_request), - t * 1000); +static void ERROR_EXIT(const char *msg) { perror(msg); exit(-1); } - printf("Spent %.4f seconds to insert %"PRIu64" records with %d record(s) per request: %.2f records/second\n", - t, (int64_t)ntables * nrecords_per_table, nrecords_per_request, - (int64_t)ntables * nrecords_per_table / t); +#ifndef TAOSDEMO_COMMIT_SHA1 +#define TAOSDEMO_COMMIT_SHA1 "unknown" +#endif - int64_t totalDelay = 0; - int64_t maxDelay = 0; - int64_t minDelay = INT16_MAX; - int64_t cntDelay = 0; - double avgDelay = 0; - for (int i = 0; i < threads; i++) { - info *t_info = infos + i; - taos_close(t_info->taos); - tsem_destroy(&(t_info->mutex_sem)); - tsem_destroy(&(t_info->lock_sem)); +#ifndef TD_VERNUMBER +#define TD_VERNUMBER "unknown" +#endif - totalDelay += t_info->totalDelay; - cntDelay += t_info->cntDelay; - if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; - if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; - } - avgDelay = (double)totalDelay / cntDelay; +#ifndef TAOSDEMO_STATUS +#define TAOSDEMO_STATUS "unknown" +#endif - fprintf(fp, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n", - avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); +static void printVersion() { + char tdengine_ver[] = TD_VERNUMBER; + char taosdemo_ver[] = TAOSDEMO_COMMIT_SHA1; + char taosdemo_status[] = TAOSDEMO_STATUS; - printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", - avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); + if (strlen(taosdemo_status) == 0) { + printf("taosdemo verison %s-%s\n", + tdengine_ver, taosdemo_ver); + } else { + printf("taosdemo verison %s-%s, status:%s\n", + tdengine_ver, taosdemo_ver, taosdemo_status); + } +} - free(pids); - free(infos); - fclose(fp); +static void printHelp() { + char indent[10] = " "; + printf("%s%s%s%s\n", indent, "-f", indent, + "The meta file to the execution procedure. Default is './meta.json'."); + printf("%s%s%s%s\n", indent, "-u", indent, + "The TDengine user name to use when connecting to the server. Default is 'root'."); +#ifdef _TD_POWER_ + printf("%s%s%s%s\n", indent, "-P", indent, + "The password to use when connecting to the server. Default is 'powerdb'."); + printf("%s%s%s%s\n", indent, "-c", indent, + "Configuration directory. Default is '/etc/power/'."); +#else + printf("%s%s%s%s\n", indent, "-P", indent, + "The password to use when connecting to the server. Default is 'taosdata'."); + printf("%s%s%s%s\n", indent, "-c", indent, + "Configuration directory. Default is '/etc/taos/'."); +#endif + printf("%s%s%s%s\n", indent, "-h", indent, + "The host to connect to TDengine. Default is localhost."); + printf("%s%s%s%s\n", indent, "-p", indent, + "The TCP/IP port number to use for the connection. Default is 0."); + printf("%s%s%s%s\n", indent, "-d", indent, + "Destination database. Default is 'test'."); + printf("%s%s%s%s\n", indent, "-a", indent, + "Set the replica parameters of the database, Default 1, min: 1, max: 3."); + printf("%s%s%s%s\n", indent, "-m", indent, + "Table prefix name. Default is 't'."); + printf("%s%s%s%s\n", indent, "-s", indent, "The select sql file."); + printf("%s%s%s%s\n", indent, "-N", indent, "Use normal table flag."); + printf("%s%s%s%s\n", indent, "-o", indent, + "Direct output to the named file. Default is './output.txt'."); + printf("%s%s%s%s\n", indent, "-q", indent, + "Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); + printf("%s%s%s%s\n", indent, "-b", indent, + "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP."); + printf("%s%s%s%s\n", indent, "-w", indent, + "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); + printf("%s%s%s%s\n", indent, "-l", indent, + "The number of columns per record. Default is 10."); + printf("%s%s%s%s\n", indent, "-T", indent, + "The number of threads. Default is 10."); + printf("%s%s%s%s\n", indent, "-i", indent, + "The sleep time (ms) between insertion. Default is 0."); + printf("%s%s%s%s\n", indent, "-r", indent, + "The number of records per request. Default is 100."); + printf("%s%s%s%s\n", indent, "-t", indent, + "The number of tables. Default is 10000."); + printf("%s%s%s%s\n", indent, "-n", indent, + "The number of records per table. Default is 10000."); + printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag."); + printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt."); + printf("%s%s%s%s\n", indent, "-O", indent, + "Insert mode--0: In order, > 0: disorder ratio. Default is in order."); + printf("%s%s%s%s\n", indent, "-R", indent, + "Out of order data's range, ms, default is 1000."); + printf("%s%s%s%s\n", indent, "-g", indent, + "Print debug info."); + printf("%s%s%s%s\n", indent, "-V, --version", indent, + "Print version info."); +/* printf("%s%s%s%s\n", indent, "-D", indent, + "if elete database if exists. 0: no, 1: yes, default is 1"); + */ +} - if (method_of_delete != 0) - { - TAOS *dtaos = taos_connect(ip_addr, user, pass, db_name, port); - double dts = getCurrentTime(); - printf("Deleteing %d table(s)......\n", ntables); - - switch (method_of_delete) - { - case 1: - // delete by table - /* Create all the tables; */ - for (int i = 0; i < ntables; i++) { - sprintf(command, "drop table %s.%s%d;", db_name, tb_prefix, i); - queryDB(dtaos, command); +static void parse_args(int argc, char *argv[], SArguments *arguments) { + char **sptr; + wordexp_t full_path; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-f") == 0) { + arguments->metaFile = argv[++i]; + } else if (strcmp(argv[i], "-c") == 0) { + char *configPath = argv[++i]; + if (wordexp(configPath, &full_path, 0) != 0) { + errorPrint( "Invalid path %s\n", configPath); + return; } - break; - case 2: - // delete by stable - if (!use_metric) { - break; + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); + + } else if (strcmp(argv[i], "-h") == 0) { + arguments->host = argv[++i]; + } else if (strcmp(argv[i], "-p") == 0) { + arguments->port = atoi(argv[++i]); + } else if (strcmp(argv[i], "-u") == 0) { + arguments->user = argv[++i]; + } else if (strcmp(argv[i], "-P") == 0) { + arguments->password = argv[++i]; + } else if (strcmp(argv[i], "-o") == 0) { + arguments->output_file = argv[++i]; + } else if (strcmp(argv[i], "-s") == 0) { + arguments->sqlFile = argv[++i]; + } else if (strcmp(argv[i], "-q") == 0) { + arguments->mode = atoi(argv[++i]); + } else if (strcmp(argv[i], "-T") == 0) { + arguments->num_of_threads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + arguments->insert_interval = atoi(argv[++i]); + } else if (strcmp(argv[i], "-qt") == 0) { + arguments->query_times = atoi(argv[++i]); + } else if (strcmp(argv[i], "-B") == 0) { + arguments->interlace_rows = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + arguments->num_of_RPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + arguments->num_of_tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + arguments->num_of_DPT = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0) { + arguments->database = argv[++i]; + } else if (strcmp(argv[i], "-l") == 0) { + arguments->num_of_CPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + sptr = arguments->datatype; + ++i; + if (strstr(argv[i], ",") == NULL) { + // only one col + if (strcasecmp(argv[i], "INT") + && strcasecmp(argv[i], "FLOAT") + && strcasecmp(argv[i], "TINYINT") + && strcasecmp(argv[i], "BOOL") + && strcasecmp(argv[i], "SMALLINT") + && strcasecmp(argv[i], "BIGINT") + && strcasecmp(argv[i], "DOUBLE") + && strcasecmp(argv[i], "BINARY") + && strcasecmp(argv[i], "NCHAR")) { + printHelp(); + ERROR_EXIT( "Invalid data_type!\n"); + exit(EXIT_FAILURE); + } + sptr[0] = argv[i]; + } else { + // more than one col + int index = 0; + char *dupstr = strdup(argv[i]); + char *running = dupstr; + char *token = strsep(&running, ","); + while (token != NULL) { + if (strcasecmp(token, "INT") + && strcasecmp(token, "FLOAT") + && strcasecmp(token, "TINYINT") + && strcasecmp(token, "BOOL") + && strcasecmp(token, "SMALLINT") + && strcasecmp(token, "BIGINT") + && strcasecmp(token, "DOUBLE") + && strcasecmp(token, "BINARY") + && strcasecmp(token, "NCHAR")) { + printHelp(); + ERROR_EXIT("Invalid data_type!\n"); + exit(EXIT_FAILURE); + } + sptr[index++] = token; + token = strsep(&running, ","); + if (index >= MAX_NUM_DATATYPE) break; + } + sptr[index] = NULL; } - else - { - sprintf(command, "drop table %s.meters;", db_name); - queryDB(dtaos, command); + } else if (strcmp(argv[i], "-w") == 0) { + arguments->len_of_binary = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + arguments->tb_prefix = argv[++i]; + } else if (strcmp(argv[i], "-N") == 0) { + arguments->use_metric = false; + } else if (strcmp(argv[i], "-x") == 0) { + arguments->insert_only = false; + } else if (strcmp(argv[i], "-y") == 0) { + arguments->answer_yes = true; + } else if (strcmp(argv[i], "-g") == 0) { + arguments->debug_print = true; + } else if (strcmp(argv[i], "-gg") == 0) { + arguments->verbose_print = true; + } else if (strcmp(argv[i], "-pp") == 0) { + arguments->performance_print = true; + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-O") == 0) { + arguments->disorderRatio = atoi(argv[++i]); + if (arguments->disorderRatio > 1 + || arguments->disorderRatio < 0) { + arguments->disorderRatio = 0; + } else if (arguments->disorderRatio == 1) { + arguments->disorderRange = 10; } - break; - case 3: - // delete by database - sprintf(command, "drop database %s;", db_name); - queryDB(dtaos, command); - break; - default: - break; + } else if (strcmp(argv[i], "-R") == 0) { + arguments->disorderRange = atoi(argv[++i]); + if (arguments->disorderRange == 1 + && (arguments->disorderRange > 50 + || arguments->disorderRange <= 0)) { + arguments->disorderRange = 10; + } + } else if (strcmp(argv[i], "-a") == 0) { + arguments->replica = atoi(argv[++i]); + if (arguments->replica > 3 || arguments->replica < 1) { + arguments->replica = 1; + } + } else if (strcmp(argv[i], "-D") == 0) { + arguments->method_of_delete = atoi(argv[++i]); + if (arguments->method_of_delete < 0 + || arguments->method_of_delete > 3) { + arguments->method_of_delete = 0; + } + } else if ((strcmp(argv[i], "--version") == 0) || + (strcmp(argv[i], "-V") == 0)){ + printVersion(); + exit(0); + } else if (strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else { + printHelp(); + ERROR_EXIT("ERROR: wrong options\n"); + exit(EXIT_FAILURE); } + } - printf("Table(s) droped!\n"); - taos_close(dtaos); + if (((arguments->debug_print) && (arguments->metaFile == NULL)) + || arguments->verbose_print) { + printf("###################################################################\n"); + printf("# meta file: %s\n", arguments->metaFile); + printf("# Server IP: %s:%hu\n", + arguments->host == NULL ? "localhost" : arguments->host, + arguments->port ); + printf("# User: %s\n", arguments->user); + printf("# Password: %s\n", arguments->password); + printf("# Use metric: %s\n", arguments->use_metric ? "true" : "false"); + if (*(arguments->datatype)) { + printf("# Specified data type: "); + for (int i = 0; i < MAX_NUM_DATATYPE; i++) + if (arguments->datatype[i]) + printf("%s,", arguments->datatype[i]); + else + break; + printf("\n"); + } + printf("# Insertion interval: %d\n", arguments->insert_interval); + printf("# Number of records per req: %d\n", arguments->num_of_RPR); + printf("# Max SQL length: %d\n", arguments->max_sql_len); + printf("# Length of Binary: %d\n", arguments->len_of_binary); + printf("# Number of Threads: %d\n", arguments->num_of_threads); + printf("# Number of Tables: %d\n", arguments->num_of_tables); + printf("# Number of Data per Table: %d\n", arguments->num_of_DPT); + printf("# Database name: %s\n", arguments->database); + printf("# Table prefix: %s\n", arguments->tb_prefix); + if (arguments->disorderRatio) { + printf("# Data order: %d\n", arguments->disorderRatio); + printf("# Data out of order rate: %d\n", arguments->disorderRange); - double dt = getCurrentTime() - dts; - printf("Spent %.4f seconds to drop %d tables\n", dt, ntables); + } + printf("# Delete method: %d\n", arguments->method_of_delete); + printf("# Answer yes when prompt: %d\n", arguments->answer_yes); + printf("# Print debug info: %d\n", arguments->debug_print); + printf("# Print verbose info: %d\n", arguments->verbose_print); + printf("###################################################################\n"); + if (!arguments->answer_yes) { + printf("Press enter key to continue\n\n"); + (void) getchar(); + } + } +} - FILE *fp = fopen(arguments.output_file, "a"); - fprintf(fp, "Spent %.4f seconds to drop %d tables\n", dt, ntables); +static bool getInfoFromJsonFile(char* file); +//static int generateOneRowDataForStb(SSuperTable* stbInfo); +//static int getDataIntoMemForStb(SSuperTable* stbInfo); +static void init_rand_data(); +static void tmfclose(FILE *fp) { + if (NULL != fp) { fclose(fp); + } +} +static void tmfree(char *buf) { + if (NULL != buf) { + free(buf); } - +} - if (false == insert_only) { - // query data - pthread_t read_id; - info *rInfo = malloc(sizeof(info)); - rInfo->start_time = 1500000000000; - rInfo->start_table_id = 0; - rInfo->end_table_id = ntables - 1; - rInfo->do_aggreFunc = do_aggreFunc; - rInfo->nrecords_per_table = nrecords_per_table; - rInfo->taos = taos_connect(ip_addr, user, pass, db_name, port); - strcpy(rInfo->tb_prefix, tb_prefix); - strcpy(rInfo->fp, arguments.output_file); +static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { + int i; + TAOS_RES *res = NULL; + int32_t code = -1; - if (!use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); - } else { - pthread_create(&read_id, NULL, readMetric, rInfo); + for (i = 0; i < 5; i++) { + if (NULL != res) { + taos_free_result(res); + res = NULL; + } + + res = taos_query(taos, command); + code = taos_errno(res); + if (0 == code) { + break; + } + } + + if (code != 0) { + if (!quiet) { + debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); + errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res)); } - pthread_join(read_id, NULL); - taos_close(rInfo->taos); + taos_free_result(res); + //taos_close(taos); + return -1; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; } - taos_cleanup(); + taos_free_result(res); return 0; } -#define MAX_SQL_SIZE 65536 -void selectSql(TAOS* taos, char* sqlcmd) -{ - TAOS_RES *pSql = taos_query(taos, sqlcmd); - int32_t code = taos_errno(pSql); - - if (code != 0) { - printf("Failed to sqlcmd:%s, reason:%s\n", sqlcmd, taos_errstr(pSql)); - taos_free_result(pSql); - exit(1); +static void getResult(TAOS_RES *res, char* resultFileName) { + TAOS_ROW row = NULL; + int num_rows = 0; + int num_fields = taos_field_count(res); + TAOS_FIELD *fields = taos_fetch_fields(res); + + FILE *fp = NULL; + if (resultFileName[0] != 0) { + fp = fopen(resultFileName, "at"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n", + __func__, __LINE__, resultFileName); + } } - - int count = 0; - while (taos_fetch_row(pSql) != NULL) { - count++; + + char* databuf = (char*) calloc(1, 100*1024*1024); + if (databuf == NULL) { + errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", + __func__, __LINE__); + if (fp) + fclose(fp); + return ; } - - taos_free_result(pSql); - return; -} + int totalLen = 0; + char temp[16000]; -/* Function to do regular expression check */ -static int regexMatch(const char *s, const char *reg, int cflags) { - regex_t regex; - char msgbuf[100] = {0}; + // fetch the records row by row + while ((row = taos_fetch_row(res))) { + if (totalLen >= 100*1024*1024 - 32000) { + if (fp) fprintf(fp, "%s", databuf); + totalLen = 0; + memset(databuf, 0, 100*1024*1024); + } + num_rows++; + int len = taos_print_row(temp, row, fields, num_fields); + len += sprintf(temp + len, "\n"); + //printf("query result:%s\n", temp); + memcpy(databuf + totalLen, temp, len); + totalLen += len; + } - /* Compile regular expression */ - if (regcomp(®ex, reg, cflags) != 0) { - printf("Fail to compile regex\n"); + if (fp) fprintf(fp, "%s", databuf); + tmfclose(fp); + free(databuf); +} + +static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { + TAOS_RES *res = taos_query(taos, command); + if (res == NULL || taos_errno(res) != 0) { + printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res)); + taos_free_result(res); + return; + } + + getResult(res, resultFileName); + taos_free_result(res); +} + +static double getCurrentTime() { + struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) { + perror("Failed to get current time in ms"); + return 0.0; + } + + return tv.tv_sec + tv.tv_usec / 1E6; +} + +static int32_t rand_bool(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 2; +} + +static int32_t rand_tinyint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 128; +} + +static int32_t rand_smallint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor] % 32767; +} + +static int32_t rand_int(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randint[cursor]; +} + +static int64_t rand_bigint(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randbigint[cursor]; +} + +static float rand_float(){ + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randfloat[cursor]; +} + +static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; +static void rand_string(char *str, int size) { + str[0] = 0; + if (size > 0) { + //--size; + int n; + for (n = 0; n < size - 1; n++) { + int key = rand_tinyint() % (int)(sizeof(charset) - 1); + str[n] = charset[key]; + } + str[n] = 0; + } +} + +static double rand_double() { + static int cursor; + cursor++; + cursor = cursor % MAX_PREPARED_RAND; + return randdouble[cursor]; + +} + +static void init_rand_data() { + for (int i = 0; i < MAX_PREPARED_RAND; i++){ + randint[i] = (int)(taosRandom() % 65535); + randbigint[i] = (int64_t)(taosRandom() % 2147483648); + randfloat[i] = (float)(taosRandom() / 1000.0); + randdouble[i] = (double)(taosRandom() / 1000000.0); + } +} + +#define SHOW_PARSE_RESULT_START() \ + do { if (g_args.metaFile) \ + printf("\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_END() \ + do { if (g_args.metaFile) \ + printf("\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_START_TO_FILE(fp) \ + do { if (g_args.metaFile) \ + fprintf(fp, "\033[1m\033[40;32m================ %s parse result START ================\033[0m\n", \ + g_args.metaFile); } while(0) + +#define SHOW_PARSE_RESULT_END_TO_FILE(fp) \ + do { if (g_args.metaFile) \ + fprintf(fp, "\033[1m\033[40;32m================ %s parse result END================\033[0m\n", \ + g_args.metaFile); } while(0) + +static int printfInsertMeta() { + SHOW_PARSE_RESULT_START(); + + printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); + printf("user: \033[33m%s\033[0m\n", g_Dbs.user); + printf("password: \033[33m%s\033[0m\n", g_Dbs.password); + printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); + printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); + printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); + printf("top insert interval: \033[33m%d\033[0m\n", g_args.insert_interval); + printf("number of records per req: \033[33m%d\033[0m\n", g_args.num_of_RPR); + printf("max sql length: \033[33m%d\033[0m\n", g_args.max_sql_len); + + printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); + + for (int i = 0; i < g_Dbs.dbCount; i++) { + printf("database[\033[33m%d\033[0m]:\n", i); + printf(" database[%d] name: \033[33m%s\033[0m\n", i, g_Dbs.db[i].dbName); + if (0 == g_Dbs.db[i].drop) { + printf(" drop: \033[33mno\033[0m\n"); + } else { + printf(" drop: \033[33myes\033[0m\n"); + } + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); + } + if (g_Dbs.db[i].dbCfg.minRows > 0) { + printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); + } + if (g_Dbs.db[i].dbCfg.quorum > 0) { + printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.precision[0] != 0) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); + } else { + printf("\033[1m\033[40;31m precision error: %s\033[0m\n", + g_Dbs.db[i].dbCfg.precision); + return -1; + } + } + + printf(" super table count: \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + printf(" super table[\033[33m%d\033[0m]:\n", j); + + printf(" stbName: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sTblName); + + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); + } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); + } else { + printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); + } + + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); + } else { + printf(" childTblExists: \033[33m%s\033[0m\n", "error"); + } + + printf(" childTblCount: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].dataSource); + printf(" insertMode: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].insertMode); + if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { + printf(" childTblLimit: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblLimit); + } + if (g_Dbs.db[i].superTbls[j].childTblOffset >= 0) { + printf(" childTblOffset: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblOffset); + } + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].insertRows); + + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); + }else { + printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); + } + printf(" interlaceRows: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].interlaceRows); + + if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { + printf(" stable insert interval: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].insertInterval); + } + + printf(" disorderRange: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].disorderRange); + printf(" disorderRatio: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].disorderRatio); + printf(" maxSqlLen: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].maxSqlLen); + printf(" timeStampStep: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].timeStampStep); + printf(" startTimestamp: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].startTimestamp); + printf(" sampleFormat: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sampleFormat); + printf(" sampleFile: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].sampleFile); + printf(" tagsFile: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].tagsFile); + printf(" columnCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "binary", 6)) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "nchar", 5))) { + printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType, + g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + printf("column[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + printf("\n"); + + printf(" tagCount: \033[33m%d\033[0m\n ", + g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "binary", strlen("binary"))) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "nchar", strlen("nchar")))) { + printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType, + g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + printf("tag[%d]:\033[33m%s\033[0m ", k, + g_Dbs.db[i].superTbls[j].tags[k].dataType); + } + } + printf("\n"); + } + printf("\n"); + } + + SHOW_PARSE_RESULT_END(); + + return 0; +} + +static void printfInsertMetaToFile(FILE* fp) { + + SHOW_PARSE_RESULT_START_TO_FILE(fp); + + fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); + fprintf(fp, "user: %s\n", g_Dbs.user); + fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); + fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); + fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); + fprintf(fp, "number of records per req: %d\n", g_args.num_of_RPR); + fprintf(fp, "max sql length: %d\n", g_args.max_sql_len); + fprintf(fp, "database count: %d\n", g_Dbs.dbCount); + + for (int i = 0; i < g_Dbs.dbCount; i++) { + fprintf(fp, "database[%d]:\n", i); + fprintf(fp, " database[%d] name: %s\n", i, g_Dbs.db[i].dbName); + if (0 == g_Dbs.db[i].drop) { + fprintf(fp, " drop: no\n"); + }else { + fprintf(fp, " drop: yes\n"); + } + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + fprintf(fp, " blocks: %d\n", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + fprintf(fp, " cache: %d\n", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + fprintf(fp, " days: %d\n", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + fprintf(fp, " keep: %d\n", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + fprintf(fp, " replica: %d\n", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + fprintf(fp, " update: %d\n", g_Dbs.db[i].dbCfg.update); + } + if (g_Dbs.db[i].dbCfg.minRows > 0) { + fprintf(fp, " minRows: %d\n", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + fprintf(fp, " maxRows: %d\n", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + fprintf(fp, " comp: %d\n", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + fprintf(fp, " walLevel: %d\n", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + fprintf(fp, " fsync: %d\n", g_Dbs.db[i].dbCfg.fsync); + } + if (g_Dbs.db[i].dbCfg.quorum > 0) { + fprintf(fp, " quorum: %d\n", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.precision[0] != 0) { + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { + fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); + } else { + fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); + } + } + + fprintf(fp, " super table count: %d\n", g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + fprintf(fp, " super table[%d]:\n", j); + + fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); + + if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + fprintf(fp, " autoCreateTable: %s\n", "no"); + } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { + fprintf(fp, " autoCreateTable: %s\n", "yes"); + } else { + fprintf(fp, " autoCreateTable: %s\n", "error"); + } + + if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + fprintf(fp, " childTblExists: %s\n", "no"); + } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { + fprintf(fp, " childTblExists: %s\n", "yes"); + } else { + fprintf(fp, " childTblExists: %s\n", "error"); + } + + fprintf(fp, " childTblCount: %d\n", + g_Dbs.db[i].superTbls[j].childTblCount); + fprintf(fp, " childTblPrefix: %s\n", + g_Dbs.db[i].superTbls[j].childTblPrefix); + fprintf(fp, " dataSource: %s\n", + g_Dbs.db[i].superTbls[j].dataSource); + fprintf(fp, " insertMode: %s\n", + g_Dbs.db[i].superTbls[j].insertMode); + fprintf(fp, " insertRows: %"PRId64"\n", + g_Dbs.db[i].superTbls[j].insertRows); + fprintf(fp, " interlace rows: %d\n", + g_Dbs.db[i].superTbls[j].interlaceRows); + if (g_Dbs.db[i].superTbls[j].interlaceRows > 0) { + fprintf(fp, " stable insert interval: %d\n", + g_Dbs.db[i].superTbls[j].insertInterval); + } + + if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { + fprintf(fp, " multiThreadWriteOneTbl: no\n"); + }else { + fprintf(fp, " multiThreadWriteOneTbl: yes\n"); + } + fprintf(fp, " interlaceRows: %d\n", + g_Dbs.db[i].superTbls[j].interlaceRows); + fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); + fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); + fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); + + fprintf(fp, " timeStampStep: %d\n", g_Dbs.db[i].superTbls[j].timeStampStep); + fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); + fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); + fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); + fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); + + fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); + if ((0 == strncasecmp( + g_Dbs.db[i].superTbls[j].columns[k].dataType, + "binary", strlen("binary"))) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, + "nchar", strlen("nchar")))) { + fprintf(fp, "column[%d]:%s(%d) ", k, + g_Dbs.db[i].superTbls[j].columns[k].dataType, + g_Dbs.db[i].superTbls[j].columns[k].dataLen); + } else { + fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); + } + } + fprintf(fp, "\n"); + + fprintf(fp, " tagCount: %d\n ", + g_Dbs.db[i].superTbls[j].tagCount); + for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { + //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "binary", strlen("binary"))) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "nchar", strlen("nchar")))) { + fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, + g_Dbs.db[i].superTbls[j].tags[k].dataLen); + } else { + fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); + } + } + fprintf(fp, "\n"); + } + fprintf(fp, "\n"); + } + + SHOW_PARSE_RESULT_END_TO_FILE(fp); +} + +static void printfQueryMeta() { + + SHOW_PARSE_RESULT_START(); + + printf("host: \033[33m%s:%u\033[0m\n", + g_queryInfo.host, g_queryInfo.port); + printf("user: \033[33m%s\033[0m\n", g_queryInfo.user); + printf("database name: \033[33m%s\033[0m\n", g_queryInfo.dbName); + + printf("\n"); + printf("specified table query info: \n"); + printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.rate); + printf("top query times:\033[33m%d\033[0m\n", g_args.query_times); + printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.sqlCount); + printf("specified tbl query times:\n"); + printf(" \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.queryTimes); + + if (SUBSCRIBE_TEST == g_args.test_mode) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + } + + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.specifiedQueryInfo.sql[i]); + } + printf("\n"); + printf("super table query info: \n"); + printf("query interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate); + printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.threadCnt); + printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.childTblCount); + printf("stable name: \033[33m%s\033[0m\n", g_queryInfo.superQueryInfo.sTblName); + printf("stb query times:\033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.queryTimes); + + if (SUBSCRIBE_TEST == g_args.test_mode) { + printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); + printf("interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); + printf("restart: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart); + printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress); + } + + printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]); + } + printf("\n"); + + SHOW_PARSE_RESULT_END(); +} + +static char* formatTimestamp(char* buf, int64_t val, int precision) { + time_t tt; + if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + } else { + tt = (time_t)(val / 1000); + } + +/* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + + struct tm* ptm = localtime(&tt); + size_t pos = strftime(buf, 32, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", (int)(val % 1000000)); + } else { + sprintf(buf + pos, ".%03d", (int)(val % 1000)); + } + + return buf; +} + +static void xDumpFieldToFile(FILE* fp, const char* val, + TAOS_FIELD* field, int32_t length, int precision) { + + if (val == NULL) { + fprintf(fp, "%s", TSDB_DATA_NULL_STR); + return; + } + + char buf[TSDB_MAX_BYTES_PER_ROW]; + switch (field->type) { + case TSDB_DATA_TYPE_BOOL: + fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + fprintf(fp, "%d", *((int8_t *)val)); + break; + case TSDB_DATA_TYPE_SMALLINT: + fprintf(fp, "%d", *((int16_t *)val)); + break; + case TSDB_DATA_TYPE_INT: + fprintf(fp, "%d", *((int32_t *)val)); + break; + case TSDB_DATA_TYPE_BIGINT: + fprintf(fp, "%" PRId64, *((int64_t *)val)); + break; + case TSDB_DATA_TYPE_FLOAT: + fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); + break; + case TSDB_DATA_TYPE_DOUBLE: + fprintf(fp, "%.9f", GET_DOUBLE_VAL(val)); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(buf, val, length); + buf[length] = 0; + fprintf(fp, "\'%s\'", buf); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(buf, *(int64_t*)val, precision); + fprintf(fp, "'%s'", buf); + break; + default: + break; + } +} + +static int xDumpResultToFile(const char* fname, TAOS_RES* tres) { + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return 0; + } + + FILE* fp = fopen(fname, "at"); + if (fp == NULL) { + errorPrint("%s() LN%d, failed to open file: %s\n", __func__, __LINE__, fname); + return -1; + } + + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); + + for (int col = 0; col < num_fields; col++) { + if (col > 0) { + fprintf(fp, ","); + } + fprintf(fp, "%s", fields[col].name); + } + fputc('\n', fp); + + int numOfRows = 0; + do { + int32_t* length = taos_fetch_lengths(tres); + for (int i = 0; i < num_fields; i++) { + if (i > 0) { + fputc(',', fp); + } + xDumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision); + } + fputc('\n', fp); + + numOfRows++; + row = taos_fetch_row(tres); + } while( row != NULL); + + fclose(fp); + + return numOfRows; +} + +static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; + + res = taos_query(taos, "show databases;"); + int32_t code = taos_errno(res); + + if (code != 0) { + errorPrint( "failed to run , reason: %s\n", taos_errstr(res)); + return -1; + } + + TAOS_FIELD *fields = taos_fetch_fields(res); + + while ((row = taos_fetch_row(res)) != NULL) { + // sys database name : 'log' + if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) { + continue; + } + + dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); + if (dbInfos[count] == NULL) { + errorPrint( "failed to allocate memory for some dbInfo[%d]\n", count); + return -1; + } + + tstrncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes); + formatTimestamp(dbInfos[count]->create_time, + *(int64_t*)row[TSDB_SHOW_DB_CREATED_TIME_INDEX], + TSDB_TIME_PRECISION_MILLI); + dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); + dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); + dbInfos[count]->replica = *((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]); + dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); + dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); + + tstrncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], + fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + dbInfos[count]->cache = *((int32_t *)row[TSDB_SHOW_DB_CACHE_INDEX]); + dbInfos[count]->blocks = *((int32_t *)row[TSDB_SHOW_DB_BLOCKS_INDEX]); + dbInfos[count]->minrows = *((int32_t *)row[TSDB_SHOW_DB_MINROWS_INDEX]); + dbInfos[count]->maxrows = *((int32_t *)row[TSDB_SHOW_DB_MAXROWS_INDEX]); + dbInfos[count]->wallevel = *((int8_t *)row[TSDB_SHOW_DB_WALLEVEL_INDEX]); + dbInfos[count]->fsync = *((int32_t *)row[TSDB_SHOW_DB_FSYNC_INDEX]); + dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); + dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); + + tstrncpy(dbInfos[count]->precision, + (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], + fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); + tstrncpy(dbInfos[count]->status, (char *)row[TSDB_SHOW_DB_STATUS_INDEX], + fields[TSDB_SHOW_DB_STATUS_INDEX].bytes); + + count++; + if (count > MAX_DATABASE_COUNT) { + errorPrint( "The database count overflow than %d\n", MAX_DATABASE_COUNT); + break; + } + } + + return count; +} + +static void printfDbInfoForQueryToFile( + char* filename, SDbInfo* dbInfos, int index) { + if (filename[0] == 0) + return; + + FILE *fp = fopen(filename, "at"); + if (fp == NULL) { + errorPrint( "failed to open file: %s\n", filename); + return; + } + + fprintf(fp, "================ database[%d] ================\n", index); + fprintf(fp, "name: %s\n", dbInfos->name); + fprintf(fp, "created_time: %s\n", dbInfos->create_time); + fprintf(fp, "ntables: %d\n", dbInfos->ntables); + fprintf(fp, "vgroups: %d\n", dbInfos->vgroups); + fprintf(fp, "replica: %d\n", dbInfos->replica); + fprintf(fp, "quorum: %d\n", dbInfos->quorum); + fprintf(fp, "days: %d\n", dbInfos->days); + fprintf(fp, "keep0,keep1,keep(D): %s\n", dbInfos->keeplist); + fprintf(fp, "cache(MB): %d\n", dbInfos->cache); + fprintf(fp, "blocks: %d\n", dbInfos->blocks); + fprintf(fp, "minrows: %d\n", dbInfos->minrows); + fprintf(fp, "maxrows: %d\n", dbInfos->maxrows); + fprintf(fp, "wallevel: %d\n", dbInfos->wallevel); + fprintf(fp, "fsync: %d\n", dbInfos->fsync); + fprintf(fp, "comp: %d\n", dbInfos->comp); + fprintf(fp, "cachelast: %d\n", dbInfos->cachelast); + fprintf(fp, "precision: %s\n", dbInfos->precision); + fprintf(fp, "update: %d\n", dbInfos->update); + fprintf(fp, "status: %s\n", dbInfos->status); + fprintf(fp, "\n"); + + fclose(fp); +} + +static void printfQuerySystemInfo(TAOS * taos) { + char filename[MAX_QUERY_SQL_LENGTH+1] = {0}; + char buffer[MAX_QUERY_SQL_LENGTH+1] = {0}; + TAOS_RES* res; + + time_t t; + struct tm* lt; + time(&t); + lt = localtime(&t); + snprintf(filename, MAX_QUERY_SQL_LENGTH, "querySystemInfo-%d-%d-%d %d:%d:%d", + lt->tm_year+1900, lt->tm_mon, lt->tm_mday, lt->tm_hour, lt->tm_min, + lt->tm_sec); + + // show variables + res = taos_query(taos, "show variables;"); + //getResult(res, filename); + xDumpResultToFile(filename, res); + + // show dnodes + res = taos_query(taos, "show dnodes;"); + xDumpResultToFile(filename, res); + //getResult(res, filename); + + // show databases + res = taos_query(taos, "show databases;"); + SDbInfo** dbInfos = (SDbInfo **)calloc(MAX_DATABASE_COUNT, sizeof(SDbInfo *)); + if (dbInfos == NULL) { + errorPrint("%s() LN%d, failed to allocate memory\n", __func__, __LINE__); + return; + } + int dbCount = getDbFromServer(taos, dbInfos); + if (dbCount <= 0) { + free(dbInfos); + return; + } + + for (int i = 0; i < dbCount; i++) { + // printf database info + printfDbInfoForQueryToFile(filename, dbInfos[i], i); + + // show db.vgroups + snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.vgroups;", dbInfos[i]->name); + res = taos_query(taos, buffer); + xDumpResultToFile(filename, res); + + // show db.stables + snprintf(buffer, MAX_QUERY_SQL_LENGTH, "show %s.stables;", dbInfos[i]->name); + res = taos_query(taos, buffer); + xDumpResultToFile(filename, res); + + free(dbInfos[i]); + } + + free(dbInfos); +} + +static int postProceSql(char* host, uint16_t port, char* sqlstr) +{ + char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; + + char *url = "/rest/sql"; + + struct hostent *server; + struct sockaddr_in serv_addr; + int bytes, sent, received, req_str_len, resp_len; + char *request_buf; + char response_buf[RESP_BUF_LEN]; + uint16_t rest_port = port + TSDB_PORT_HTTP; + + int req_buf_len = strlen(sqlstr) + REQ_EXTRA_BUF_LEN; + + request_buf = malloc(req_buf_len); + if (NULL == request_buf) + ERROR_EXIT("ERROR, cannot allocate memory."); + + char userpass_buf[INPUT_BUF_LEN]; + int mod_table[] = {0, 2, 1}; + + static char base64[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/'}; + + snprintf(userpass_buf, INPUT_BUF_LEN, "%s:%s", + g_Dbs.user, g_Dbs.password); + size_t userpass_buf_len = strlen(userpass_buf); + size_t encoded_len = 4 * ((userpass_buf_len +2) / 3); + + char base64_buf[INPUT_BUF_LEN]; +#ifdef WINDOWS + WSADATA wsaData; + WSAStartup(MAKEWORD(2, 2), &wsaData); + SOCKET sockfd; +#else + int sockfd; +#endif + sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { +#ifdef WINDOWS + errorPrint( "Could not create socket : %d" , WSAGetLastError()); +#endif + debugPrint("%s() LN%d, sockfd=%d\n", __func__, __LINE__, sockfd); + free(request_buf); + ERROR_EXIT("ERROR opening socket"); + } + + server = gethostbyname(host); + if (server == NULL) { + free(request_buf); + ERROR_EXIT("ERROR, no such host"); + } + + debugPrint("h_name: %s\nh_addretype: %s\nh_length: %d\n", + server->h_name, + (server->h_addrtype == AF_INET)?"ipv4":"ipv6", + server->h_length); + + memset(&serv_addr, 0, sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = htons(rest_port); +#ifdef WINDOWS + serv_addr.sin_addr.s_addr = inet_addr(host); +#else + memcpy(&serv_addr.sin_addr.s_addr,server->h_addr,server->h_length); +#endif + + int retConn = connect(sockfd,(struct sockaddr *)&serv_addr,sizeof(serv_addr)); + debugPrint("%s() LN%d connect() return %d\n", __func__, __LINE__, retConn); + if (retConn < 0) { + free(request_buf); + ERROR_EXIT("ERROR connecting"); + } + + memset(base64_buf, 0, INPUT_BUF_LEN); + + for (int n = 0, m = 0; n < userpass_buf_len;) { + uint32_t oct_a = n < userpass_buf_len ? + (unsigned char) userpass_buf[n++]:0; + uint32_t oct_b = n < userpass_buf_len ? + (unsigned char) userpass_buf[n++]:0; + uint32_t oct_c = n < userpass_buf_len ? + (unsigned char) userpass_buf[n++]:0; + uint32_t triple = (oct_a << 0x10) + (oct_b << 0x08) + oct_c; + + base64_buf[m++] = base64[(triple >> 3* 6) & 0x3f]; + base64_buf[m++] = base64[(triple >> 2* 6) & 0x3f]; + base64_buf[m++] = base64[(triple >> 1* 6) & 0x3f]; + base64_buf[m++] = base64[(triple >> 0* 6) & 0x3f]; + } + + for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++) + base64_buf[encoded_len - 1 - l] = '='; + + debugPrint("%s() LN%d: auth string base64 encoded: %s\n", + __func__, __LINE__, base64_buf); + char *auth = base64_buf; + + int r = snprintf(request_buf, + req_buf_len, + req_fmt, url, host, rest_port, + auth, strlen(sqlstr), sqlstr); + if (r >= req_buf_len) { + free(request_buf); + ERROR_EXIT("ERROR too long request"); + } + verbosePrint("%s() LN%d: Request:\n%s\n", __func__, __LINE__, request_buf); + + req_str_len = strlen(request_buf); + sent = 0; + do { +#ifdef WINDOWS + bytes = send(sockfd, request_buf + sent, req_str_len - sent, 0); +#else + bytes = write(sockfd, request_buf + sent, req_str_len - sent); +#endif + if (bytes < 0) + ERROR_EXIT("ERROR writing message to socket"); + if (bytes == 0) + break; + sent+=bytes; + } while (sent < req_str_len); + + memset(response_buf, 0, RESP_BUF_LEN); + resp_len = sizeof(response_buf) - 1; + received = 0; + do { +#ifdef WINDOWS + bytes = recv(sockfd, response_buf + received, resp_len - received, 0); +#else + bytes = read(sockfd, response_buf + received, resp_len - received); +#endif + if (bytes < 0) { + free(request_buf); + ERROR_EXIT("ERROR reading response from socket"); + } + if (bytes == 0) + break; + received += bytes; + } while (received < resp_len); + + if (received == resp_len) { + free(request_buf); + ERROR_EXIT("ERROR storing complete response from socket"); + } + + response_buf[RESP_BUF_LEN - 1] = '\0'; + printf("Response:\n%s\n", response_buf); + + free(request_buf); +#ifdef WINDOWS + closesocket(sockfd); + WSACleanup(); +#else + close(sockfd); +#endif + + return 0; +} + +static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) { + char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); + if (NULL == dataBuf) { + errorPrint("%s() LN%d, calloc failed! size:%d\n", __func__, __LINE__, TSDB_MAX_SQL_LEN+1); + return NULL; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos); + + return dataBuf; +} + +static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) { + char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); + if (NULL == dataBuf) { + printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); + return NULL; + } + + int dataLen = 0; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "("); + for (int i = 0; i < stbInfo->tagCount; i++) { + if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", strlen("binary"))) + || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", strlen("nchar")))) { + if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) { + printf("binary or nchar length overflow, max size:%u\n", + (uint32_t)TSDB_MAX_BINARY_LEN); + tmfree(dataBuf); + return NULL; + } + + int tagBufLen = stbInfo->tags[i].dataLen + 1; + char* buf = (char*)calloc(tagBufLen, 1); + if (NULL == buf) { + printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); + tmfree(dataBuf); + return NULL; + } + + if (tableSeq % 2) { + tstrncpy(buf, "beijing", tagBufLen); + } else { + tstrncpy(buf, "shanghai", tagBufLen); + } + //rand_string(buf, stbInfo->tags[i].dataLen); + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "\'%s\', ", buf); + tmfree(buf); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "int", strlen("int"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d, ", tableSeq); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "bigint", strlen("bigint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "float", strlen("float"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "double", strlen("double"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "smallint", strlen("smallint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "tinyint", strlen("tinyint"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "bool", strlen("bool"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->tags[i].dataType, + "timestamp", strlen("timestamp"))) { + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + "%"PRId64", ", rand_bigint()); + } else { + printf("No support data type: %s\n", stbInfo->tags[i].dataType); + tmfree(dataBuf); + return NULL; + } + } + + dataLen -= 2; + dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); + return dataBuf; +} + +static int calcRowLen(SSuperTable* superTbls) { + int colIndex; + int lenOfOneRow = 0; + + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + lenOfOneRow += 21; + } else { + printf("get error data type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + + int tagIndex; + int lenOfTagOfOneRow = 0; + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + printf("get error tag type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + + return 0; +} + + +static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, + char* dbName, char* sTblName, char** childTblNameOfSuperTbl, + int* childTblCountOfSuperTbl, int limit, int offset) { + + char command[BUFFER_SIZE] = "\0"; + char limitBuf[100] = "\0"; + + TAOS_RES * res; + TAOS_ROW row = NULL; + + char* childTblName = *childTblNameOfSuperTbl; + + if (offset >= 0) { + snprintf(limitBuf, 100, " limit %d offset %d", limit, offset); + } + + //get all child table name use cmd: select tbname from superTblName; + snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", + dbName, sTblName, limitBuf); + + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + taos_free_result(res); + taos_close(taos); + errorPrint("%s() LN%d, failed to run command %s\n", + __func__, __LINE__, command); + exit(-1); + } + + int childTblCount = (limit < 0)?10000:limit; + int count = 0; + if (childTblName == NULL) { + childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); + if (NULL == childTblName) { + taos_free_result(res); + taos_close(taos); + errorPrint("%s() LN%d, failed to allocate memory!\n", __func__, __LINE__); + exit(-1); + } + } + + char* pTblName = childTblName; + while ((row = taos_fetch_row(res)) != NULL) { + int32_t* len = taos_fetch_lengths(res); + tstrncpy(pTblName, (char *)row[0], len[0]+1); + //printf("==== sub table name: %s\n", pTblName); + count++; + if (count >= childTblCount - 1) { + char *tmp = realloc(childTblName, + (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1); + if (tmp != NULL) { + childTblName = tmp; + childTblCount = (int)(childTblCount*1.5); + memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, + (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN)); + } else { + // exit, if allocate more memory failed + errorPrint("%s() LN%d, realloc fail for save child table name of %s.%s\n", + __func__, __LINE__, dbName, sTblName); + tmfree(childTblName); + taos_free_result(res); + taos_close(taos); + exit(-1); + } + } + pTblName = childTblName + count * TSDB_TABLE_NAME_LEN; + } + + *childTblCountOfSuperTbl = count; + *childTblNameOfSuperTbl = childTblName; + + taos_free_result(res); + return 0; +} + +static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, + char* sTblName, char** childTblNameOfSuperTbl, + int* childTblCountOfSuperTbl) { + + return getChildNameOfSuperTableWithLimitAndOffset(taos, dbName, sTblName, + childTblNameOfSuperTbl, childTblCountOfSuperTbl, + -1, -1); +} + +static int getSuperTableFromServer(TAOS * taos, char* dbName, + SSuperTable* superTbls) { + + char command[BUFFER_SIZE] = "\0"; + TAOS_RES * res; + TAOS_ROW row = NULL; + int count = 0; + + //get schema use cmd: describe superTblName; + snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName); + res = taos_query(taos, command); + int32_t code = taos_errno(res); + if (code != 0) { + printf("failed to run command %s\n", command); + taos_free_result(res); + return -1; + } + + int tagIndex = 0; + int columnIndex = 0; + TAOS_FIELD *fields = taos_fetch_fields(res); + while ((row = taos_fetch_row(res)) != NULL) { + if (0 == count) { + count++; + continue; + } + + if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) { + tstrncpy(superTbls->tags[tagIndex].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + tstrncpy(superTbls->tags[tagIndex].dataType, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->tags[tagIndex].dataLen = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(superTbls->tags[tagIndex].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + tagIndex++; + } else { + tstrncpy(superTbls->columns[columnIndex].field, + (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], + fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); + tstrncpy(superTbls->columns[columnIndex].dataType, + (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], + fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); + superTbls->columns[columnIndex].dataLen = + *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); + tstrncpy(superTbls->columns[columnIndex].note, + (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], + fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); + columnIndex++; + } + count++; + } + + superTbls->columnCount = columnIndex; + superTbls->tagCount = tagIndex; + taos_free_result(res); + + calcRowLen(superTbls); + +/* + if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { + //get all child table name use cmd: select tbname from superTblName; + int childTblCount = 10000; + superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); + if (superTbls->childTblName == NULL) { + errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); + return -1; + } + getAllChildNameOfSuperTable(taos, dbName, + superTbls->sTblName, + &superTbls->childTblName, + &superTbls->childTblCount); + } + */ + return 0; +} + +static int createSuperTable(TAOS * taos, char* dbName, + SSuperTable* superTbls, bool use_metric) { + char command[BUFFER_SIZE] = "\0"; + + char cols[STRING_LEN] = "\0"; + int colIndex; + int len = 0; + + int lenOfOneRow = 0; + for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { + char* dataType = superTbls->columns[colIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(cols + len, STRING_LEN - len, + ", col%d %s(%d)", colIndex, "BINARY", + superTbls->columns[colIndex].dataLen); + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(cols + len, STRING_LEN - len, + ", col%d %s(%d)", colIndex, "NCHAR", + superTbls->columns[colIndex].dataLen); + lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); + lenOfOneRow += 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT"); + lenOfOneRow += 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT"); + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT"); + lenOfOneRow += 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL"); + lenOfOneRow += 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT"); + lenOfOneRow += 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE"); + lenOfOneRow += 42; + } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { + len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP"); + lenOfOneRow += 21; + } else { + taos_close(taos); + printf("config error data type : %s\n", dataType); + exit(-1); + } + } + + superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp + //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow); + + // save for creating child table + superTbls->colsOfCreateChildTable = (char*)calloc(len+20, 1); + if (NULL == superTbls->colsOfCreateChildTable) { + printf("Failed when calloc, size:%d", len+1); + taos_close(taos); + exit(-1); + } + snprintf(superTbls->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols); + verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbls->colsOfCreateChildTable); + + if (use_metric) { + char tags[STRING_LEN] = "\0"; + int tagIndex; + len = 0; + + int lenOfTagOfOneRow = 0; + len += snprintf(tags + len, STRING_LEN - len, "("); + for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { + char* dataType = superTbls->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, + "BINARY", superTbls->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, + "NCHAR", superTbls->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "INT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "BIGINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "SMALLINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "TINYINT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "BOOL"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "FLOAT"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "DOUBLE"); + lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; + } else { + taos_close(taos); + printf("config error tag type : %s\n", dataType); + exit(-1); + } + } + len -= 2; + len += snprintf(tags + len, STRING_LEN - len, ")"); + + superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; + + snprintf(command, BUFFER_SIZE, + "create table if not exists %s.%s (ts timestamp%s) tags %s", + dbName, superTbls->sTblName, cols, tags); + verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command); + + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + errorPrint( "create supertable %s failed!\n\n", + superTbls->sTblName); + return -1; + } + debugPrint("create supertable %s success!\n\n", superTbls->sTblName); + } + return 0; +} + +static int createDatabasesAndStables() { + TAOS * taos = NULL; + int ret = 0; + taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); + if (taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + return -1; + } + char command[BUFFER_SIZE] = "\0"; + + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].drop) { + sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); + verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + taos_close(taos); + return -1; + } + } + + int dataLen = 0; + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); + + if (g_Dbs.db[i].dbCfg.blocks > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.quorum > 1) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); + } + //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { + // dataLen += snprintf(command + dataLen, + // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + //} + if (g_Dbs.db[i].dbCfg.minRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.cacheLast > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " fsync %d", g_Dbs.db[i].dbCfg.fsync); + } + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, + "us", strlen("us")))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + } + + debugPrint("%s() %d command: %s\n", __func__, __LINE__, command); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + taos_close(taos); + errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); + return -1; + } + printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); + + debugPrint("%s() %d supertbl count:%d\n", + __func__, __LINE__, g_Dbs.db[i].superTblCount); + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, + g_Dbs.db[i].superTbls[j].sTblName); + verbosePrint("%s() %d command: %s\n", __func__, __LINE__, command); + + ret = queryDbExec(taos, command, NO_INSERT_TYPE, true); + + if ((ret != 0) || (g_Dbs.db[i].drop)) { + ret = createSuperTable(taos, g_Dbs.db[i].dbName, + &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); + + if (0 != ret) { + errorPrint("\ncreate super table %d failed!\n\n", j); + taos_close(taos); + return -1; + } + } + + ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, + &g_Dbs.db[i].superTbls[j]); + if (0 != ret) { + errorPrint("\nget super table %s.%s info failed!\n\n", + g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); + taos_close(taos); + return -1; + } + } + } + + taos_close(taos); + return 0; +} + +static void* createTable(void *sarg) +{ + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + int64_t lastPrintTime = taosGetTimestampMs(); + + int buff_len; + if (superTblInfo) + buff_len = superTblInfo->maxSqlLen; + else + buff_len = BUFFER_SIZE; + + char *buffer = calloc(buff_len, 1); + if (buffer == NULL) { + errorPrint("%s() LN%d, Memory allocated failed!\n", __func__, __LINE__); + exit(-1); + } + + int len = 0; + int batchNum = 0; + + verbosePrint("%s() LN%d: Creating table from %d to %d\n", + __func__, __LINE__, + winfo->start_table_from, winfo->end_table_to); + + for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) { + if (0 == g_Dbs.use_metric) { + snprintf(buffer, buff_len, + "create table if not exists %s.%s%d %s;", + winfo->db_name, + g_args.tb_prefix, i, + winfo->cols); + } else { + if (superTblInfo == NULL) { + errorPrint("%s() LN%d, use metric, but super table info is NULL\n", + __func__, __LINE__); + free(buffer); + exit(-1); + } else { + if (0 == len) { + batchNum = 0; + memset(buffer, 0, buff_len); + len += snprintf(buffer + len, + buff_len - len, "create table "); + } + + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo, i); + } else { + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + i % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + free(buffer); + return NULL; + } + + len += snprintf(buffer + len, + superTblInfo->maxSqlLen - len, + "if not exists %s.%s%d using %s.%s tags %s ", + winfo->db_name, superTblInfo->childTblPrefix, + i, winfo->db_name, + superTblInfo->sTblName, tagsValBuf); + free(tagsValBuf); + batchNum++; + + if ((batchNum < superTblInfo->batchCreateTableNum) + && ((superTblInfo->maxSqlLen - len) + >= (superTblInfo->lenOfTagOfOneRow + 256))) { + continue; + } + } + } + + len = 0; + verbosePrint("%s() LN%d %s\n", __func__, __LINE__, buffer); + if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)){ + errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer); + free(buffer); + return NULL; + } + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] already create %d - %d tables\n", + winfo->threadID, winfo->start_table_from, i); + lastPrintTime = currentPrintTime; + } + } + + if (0 != len) { + verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer); + if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)) { + errorPrint( "queryDbExec() failed. buffer:\n%s\n", buffer); + } + } + + free(buffer); + return NULL; +} + +static int startMultiThreadCreateChildTable( + char* cols, int threads, int startFrom, int ntables, + char* db_name, SSuperTable* superTblInfo) { + + pthread_t *pids = malloc(threads * sizeof(pthread_t)); + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + + if ((NULL == pids) || (NULL == infos)) { + printf("malloc failed\n"); + exit(-1); + } + + if (threads < 1) { + threads = 1; + } + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + b = ntables % threads; + + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); + t_info->superTblInfo = superTblInfo; + verbosePrint("%s() %d db_name: %s\n", __func__, __LINE__, db_name); + t_info->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + db_name, + g_Dbs.port); + if (t_info->taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); + free(pids); + free(infos); + return -1; + } + + t_info->start_table_from = startFrom; + t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = t_info->end_table_to + 1; + t_info->use_metric = true; + t_info->cols = cols; + t_info->minDelay = INT16_MAX; + pthread_create(pids + i, NULL, createTable, t_info); + } + + for (int i = 0; i < threads; i++) { + pthread_join(pids[i], NULL); + } + + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + taos_close(t_info->taos); + } + + free(pids); + free(infos); + + return 0; +} + +static void createChildTables() { + char tblColsBuf[MAX_SQL_SIZE]; + int len; + + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].superTblCount > 0) { + // with super table + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) + || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + + verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + int startFrom = 0; + g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; + + verbosePrint("%s() LN%d: create %d child tables from %d\n", + __func__, __LINE__, g_totalChildTables, startFrom); + startMultiThreadCreateChildTable( + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, + g_Dbs.threadCountByCreateTbl, + startFrom, + g_Dbs.db[i].superTbls[j].childTblCount, + g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); + } + } else { + // normal table + len = snprintf(tblColsBuf, MAX_SQL_SIZE, "(TS TIMESTAMP"); + int j = 0; + while (g_args.datatype[j]) { + if ((strncasecmp(g_args.datatype[j], "BINARY", strlen("BINARY")) == 0) + || (strncasecmp(g_args.datatype[j], + "NCHAR", strlen("NCHAR")) == 0)) { + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, + ", COL%d %s(60)", j, g_args.datatype[j]); + } else { + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, + ", COL%d %s", j, g_args.datatype[j]); + } + len = strlen(tblColsBuf); + j++; + } + + len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); + + verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n", + __func__, __LINE__, + g_Dbs.db[i].dbName, g_args.num_of_tables, tblColsBuf); + startMultiThreadCreateChildTable( + tblColsBuf, + g_Dbs.threadCountByCreateTbl, + 0, + g_args.num_of_tables, + g_Dbs.db[i].dbName, + NULL); + } + } +} + +/* + Read 10000 lines at most. If more than 10000 lines, continue to read after using +*/ +static int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + + FILE *fp = fopen(superTblInfo->tagsFile, "r"); + if (fp == NULL) { + printf("Failed to open tags file: %s, reason:%s\n", + superTblInfo->tagsFile, strerror(errno)); + return -1; + } + + if (superTblInfo->tagDataBuf) { + free(superTblInfo->tagDataBuf); + superTblInfo->tagDataBuf = NULL; + } + + int tagCount = 10000; + int count = 0; + char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount); + if (tagDataBuf == NULL) { + printf("Failed to calloc, reason:%s\n", strerror(errno)); + fclose(fp); + return -1; + } + + while ((readLen = tgetline(&line, &n, fp)) != -1) { + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen); + count++; + + if (count >= tagCount - 1) { + char *tmp = realloc(tagDataBuf, + (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow); + if (tmp != NULL) { + tagDataBuf = tmp; + tagCount = (int)(tagCount*1.5); + memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow, + 0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow)); + } else { + // exit, if allocate more memory failed + printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile); + tmfree(tagDataBuf); + free(line); + fclose(fp); + return -1; + } + } + } + + superTblInfo->tagDataBuf = tagDataBuf; + superTblInfo->tagSampleCount = count; + + free(line); + fclose(fp); + return 0; +} + +int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { + // TODO + return 0; +} + +/* + Read 10000 lines at most. If more than 10000 lines, continue to read after using +*/ +static int readSampleFromCsvFileToMem( + SSuperTable* superTblInfo) { + size_t n = 0; + ssize_t readLen = 0; + char * line = NULL; + int getRows = 0; + + FILE* fp = fopen(superTblInfo->sampleFile, "r"); + if (fp == NULL) { + errorPrint( "Failed to open sample file: %s, reason:%s\n", + superTblInfo->sampleFile, strerror(errno)); + return -1; + } + + assert(superTblInfo->sampleDataBuf); + memset(superTblInfo->sampleDataBuf, 0, + MAX_SAMPLES_ONCE_FROM_FILE * superTblInfo->lenOfOneRow); + while (1) { + readLen = tgetline(&line, &n, fp); + if (-1 == readLen) { + if(0 != fseek(fp, 0, SEEK_SET)) { + errorPrint( "Failed to fseek file: %s, reason:%s\n", + superTblInfo->sampleFile, strerror(errno)); + fclose(fp); + return -1; + } + continue; + } + + if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { + line[--readLen] = 0; + } + + if (readLen == 0) { + continue; + } + + if (readLen > superTblInfo->lenOfOneRow) { + printf("sample row len[%d] overflow define schema len[%d], so discard this row\n", + (int32_t)readLen, superTblInfo->lenOfOneRow); + continue; + } + + verbosePrint("readLen=%ld stb->lenOfOneRow=%d getRows=%d\n", (long)readLen, + superTblInfo->lenOfOneRow, getRows); + + memcpy(superTblInfo->sampleDataBuf + getRows * superTblInfo->lenOfOneRow, + line, readLen); + getRows++; + + if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) { + break; + } + } + + fclose(fp); + tmfree(line); + return 0; +} + +static bool getColumnAndTagTypeFromInsertJsonFile( + cJSON* stbInfo, SSuperTable* superTbls) { + bool ret = false; + + // columns + cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); + if (columns && columns->type != cJSON_Array) { + printf("ERROR: failed to read json, columns not found\n"); + goto PARSE_OVER; + } else if (NULL == columns) { + superTbls->columnCount = 0; + superTbls->tagCount = 0; + return true; + } + + int columnSize = cJSON_GetArraySize(columns); + if (columnSize > MAX_COLUMN_COUNT) { + errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", + __func__, __LINE__, MAX_COLUMN_COUNT); + goto PARSE_OVER; + } + + int count = 1; + int index = 0; + StrColumn columnCase; + + //superTbls->columnCount = columnSize; + for (int k = 0; k < columnSize; ++k) { + cJSON* column = cJSON_GetArrayItem(columns, k); + if (column == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(column, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + errorPrint("%s() LN%d, failed to read json, column count not found\n", __func__, __LINE__); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(column, "type"); + if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { + errorPrint("%s() LN%d: failed to read json, column type not found\n", __func__, __LINE__); + goto PARSE_OVER; + } + //tstrncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + + cJSON* dataLen = cJSON_GetObjectItem(column, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + debugPrint("%s() LN%d: failed to read json, column len not found\n", __func__, __LINE__); + goto PARSE_OVER; + } else { + columnCase.dataLen = 8; + } + + for (int n = 0; n < count; ++n) { + tstrncpy(superTbls->columns[index].dataType, + columnCase.dataType, MAX_TB_NAME_SIZE); + superTbls->columns[index].dataLen = columnCase.dataLen; + index++; + } + } + superTbls->columnCount = index; + + count = 1; + index = 0; + // tags + cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); + if (!tags || tags->type != cJSON_Array) { + debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__); + goto PARSE_OVER; + } + + int tagSize = cJSON_GetArraySize(tags); + if (tagSize > MAX_TAG_COUNT) { + debugPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", __func__, __LINE__, MAX_TAG_COUNT); + goto PARSE_OVER; + } + + //superTbls->tagCount = tagSize; + for (int k = 0; k < tagSize; ++k) { + cJSON* tag = cJSON_GetArrayItem(tags, k); + if (tag == NULL) continue; + + count = 1; + cJSON* countObj = cJSON_GetObjectItem(tag, "count"); + if (countObj && countObj->type == cJSON_Number) { + count = countObj->valueint; + } else if (countObj && countObj->type != cJSON_Number) { + printf("ERROR: failed to read json, column count not found\n"); + goto PARSE_OVER; + } else { + count = 1; + } + + // column info + memset(&columnCase, 0, sizeof(StrColumn)); + cJSON *dataType = cJSON_GetObjectItem(tag, "type"); + if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { + printf("ERROR: failed to read json, tag type not found\n"); + goto PARSE_OVER; + } + tstrncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); + + cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); + if (dataLen && dataLen->type == cJSON_Number) { + columnCase.dataLen = dataLen->valueint; + } else if (dataLen && dataLen->type != cJSON_Number) { + printf("ERROR: failed to read json, column len not found\n"); + goto PARSE_OVER; + } else { + columnCase.dataLen = 0; + } + + for (int n = 0; n < count; ++n) { + tstrncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); + superTbls->tags[index].dataLen = columnCase.dataLen; + index++; + } + } + superTbls->tagCount = index; + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getMetaFromInsertJsonFile(cJSON* root) { + bool ret = false; + + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + tstrncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } + + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + tstrncpy(g_Dbs.host, host->valuestring, MAX_HOSTNAME_SIZE); + } else if (!host) { + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + } else { + printf("ERROR: failed to read json, host not found\n"); + goto PARSE_OVER; + } + + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_Dbs.port = port->valueint; + } else if (!port) { + g_Dbs.port = 6030; + } + + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + tstrncpy(g_Dbs.user, user->valuestring, MAX_USERNAME_SIZE); + } else if (!user) { + tstrncpy(g_Dbs.user, "root", MAX_USERNAME_SIZE); + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + tstrncpy(g_Dbs.password, password->valuestring, MAX_PASSWORD_SIZE); + } else if (!password) { + tstrncpy(g_Dbs.password, "taosdata", MAX_PASSWORD_SIZE); + } + + cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); + if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { + tstrncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); + } else if (!resultfile) { + tstrncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); + } + + cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); + if (threads && threads->type == cJSON_Number) { + g_Dbs.threadCount = threads->valueint; + } else if (!threads) { + g_Dbs.threadCount = 1; + } else { + printf("ERROR: failed to read json, threads not found\n"); + goto PARSE_OVER; + } + + cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); + if (threads2 && threads2->type == cJSON_Number) { + g_Dbs.threadCountByCreateTbl = threads2->valueint; + } else if (!threads2) { + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + } else { + errorPrint("%s() LN%d, failed to read json, threads2 not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* gInsertInterval = cJSON_GetObjectItem(root, "insert_interval"); + if (gInsertInterval && gInsertInterval->type == cJSON_Number) { + g_args.insert_interval = gInsertInterval->valueint; + } else if (!gInsertInterval) { + g_args.insert_interval = 0; + } else { + errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* interlaceRows = cJSON_GetObjectItem(root, "interlace_rows"); + if (interlaceRows && interlaceRows->type == cJSON_Number) { + g_args.interlace_rows = interlaceRows->valueint; + + // rows per table need be less than insert batch + if (g_args.interlace_rows > g_args.num_of_RPR) { + printf("NOTICE: interlace rows value %d > num_of_records_per_request %d\n\n", + g_args.interlace_rows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_request %d\n\n", + g_args.num_of_RPR); + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + g_args.interlace_rows = g_args.num_of_RPR; + } + } else if (!interlaceRows) { + g_args.interlace_rows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req + } else { + errorPrint("%s() LN%d, failed to read json, interlace_rows input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* maxSqlLen = cJSON_GetObjectItem(root, "max_sql_len"); + if (maxSqlLen && maxSqlLen->type == cJSON_Number) { + g_args.max_sql_len = maxSqlLen->valueint; + } else if (!maxSqlLen) { + g_args.max_sql_len = TSDB_PAYLOAD_SIZE; + } else { + errorPrint("%s() LN%d, failed to read json, max_sql_len input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* numRecPerReq = cJSON_GetObjectItem(root, "num_of_records_per_req"); + if (numRecPerReq && numRecPerReq->type == cJSON_Number) { + g_args.num_of_RPR = numRecPerReq->valueint; + } else if (!numRecPerReq) { + g_args.num_of_RPR = 0xffff; + } else { + errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, + if (answerPrompt + && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { + if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { + g_args.answer_yes = false; + } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { + g_args.answer_yes = true; + } else { + g_args.answer_yes = false; + } + } else if (!answerPrompt) { + g_args.answer_yes = false; + } else { + printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + goto PARSE_OVER; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (!dbs || dbs->type != cJSON_Array) { + printf("ERROR: failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + int dbSize = cJSON_GetArraySize(dbs); + if (dbSize > MAX_DB_COUNT) { + errorPrint( + "ERROR: failed to read json, databases size overflow, max database is %d\n", + MAX_DB_COUNT); + goto PARSE_OVER; + } + + g_Dbs.dbCount = dbSize; + for (int i = 0; i < dbSize; ++i) { + cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); + if (dbinfos == NULL) continue; + + // dbinfo + cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); + if (!dbinfo || dbinfo->type != cJSON_Object) { + printf("ERROR: failed to read json, dbinfo not found\n"); + goto PARSE_OVER; + } + + cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); + if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { + printf("ERROR: failed to read json, db name not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); + + cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); + if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { + if (0 == strncasecmp(drop->valuestring, "yes", strlen("yes"))) { + g_Dbs.db[i].drop = true; + } else { + g_Dbs.db[i].drop = false; + } + } else if (!drop) { + g_Dbs.db[i].drop = g_args.drop_database; + } else { + errorPrint("%s() LN%d, failed to read json, drop input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); + if (precision && precision->type == cJSON_String + && precision->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, + MAX_DB_NAME_SIZE); + } else if (!precision) { + //tstrncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); + memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); + } else { + printf("ERROR: failed to read json, precision not found\n"); + goto PARSE_OVER; + } + + cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); + if (update && update->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.update = update->valueint; + } else if (!update) { + g_Dbs.db[i].dbCfg.update = -1; + } else { + printf("ERROR: failed to read json, update not found\n"); + goto PARSE_OVER; + } + + cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); + if (replica && replica->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.replica = replica->valueint; + } else if (!replica) { + g_Dbs.db[i].dbCfg.replica = -1; + } else { + printf("ERROR: failed to read json, replica not found\n"); + goto PARSE_OVER; + } + + cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); + if (keep && keep->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.keep = keep->valueint; + } else if (!keep) { + g_Dbs.db[i].dbCfg.keep = -1; + } else { + printf("ERROR: failed to read json, keep not found\n"); + goto PARSE_OVER; + } + + cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); + if (days && days->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.days = days->valueint; + } else if (!days) { + g_Dbs.db[i].dbCfg.days = -1; + } else { + printf("ERROR: failed to read json, days not found\n"); + goto PARSE_OVER; + } + + cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); + if (cache && cache->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cache = cache->valueint; + } else if (!cache) { + g_Dbs.db[i].dbCfg.cache = -1; + } else { + printf("ERROR: failed to read json, cache not found\n"); + goto PARSE_OVER; + } + + cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); + if (blocks && blocks->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.blocks = blocks->valueint; + } else if (!blocks) { + g_Dbs.db[i].dbCfg.blocks = -1; + } else { + printf("ERROR: failed to read json, block not found\n"); + goto PARSE_OVER; + } + + //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); + //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; + //} else if (!maxtablesPerVnode) { + // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; + //} else { + // printf("failed to read json, maxtablesPerVnode not found"); + // goto PARSE_OVER; + //} + + cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); + if (minRows && minRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.minRows = minRows->valueint; + } else if (!minRows) { + g_Dbs.db[i].dbCfg.minRows = -1; + } else { + printf("ERROR: failed to read json, minRows not found\n"); + goto PARSE_OVER; + } + + cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); + if (maxRows && maxRows->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; + } else if (!maxRows) { + g_Dbs.db[i].dbCfg.maxRows = -1; + } else { + printf("ERROR: failed to read json, maxRows not found\n"); + goto PARSE_OVER; + } + + cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); + if (comp && comp->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.comp = comp->valueint; + } else if (!comp) { + g_Dbs.db[i].dbCfg.comp = -1; + } else { + printf("ERROR: failed to read json, comp not found\n"); + goto PARSE_OVER; + } + + cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); + if (walLevel && walLevel->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; + } else if (!walLevel) { + g_Dbs.db[i].dbCfg.walLevel = -1; + } else { + printf("ERROR: failed to read json, walLevel not found\n"); + goto PARSE_OVER; + } + + cJSON* cacheLast= cJSON_GetObjectItem(dbinfo, "cachelast"); + if (cacheLast && cacheLast->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.cacheLast = cacheLast->valueint; + } else if (!cacheLast) { + g_Dbs.db[i].dbCfg.cacheLast = -1; + } else { + printf("ERROR: failed to read json, cacheLast not found\n"); + goto PARSE_OVER; + } + + cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); + if (quorum && quorum->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.quorum = quorum->valueint; + } else if (!quorum) { + g_Dbs.db[i].dbCfg.quorum = 1; + } else { + printf("failed to read json, quorum input mistake"); + goto PARSE_OVER; + } + + cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); + if (fsync && fsync->type == cJSON_Number) { + g_Dbs.db[i].dbCfg.fsync = fsync->valueint; + } else if (!fsync) { + g_Dbs.db[i].dbCfg.fsync = -1; + } else { + errorPrint("%s() LN%d, failed to read json, fsync input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + // super_talbes + cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); + if (!stables || stables->type != cJSON_Array) { + errorPrint("%s() LN%d, failed to read json, super_tables not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + int stbSize = cJSON_GetArraySize(stables); + if (stbSize > MAX_SUPER_TABLE_COUNT) { + errorPrint( + "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n", + __func__, __LINE__, MAX_SUPER_TABLE_COUNT); + goto PARSE_OVER; + } + + g_Dbs.db[i].superTblCount = stbSize; + for (int j = 0; j < stbSize; ++j) { + cJSON* stbInfo = cJSON_GetArrayItem(stables, j); + if (stbInfo == NULL) continue; + + // dbinfo + cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); + if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { + errorPrint("%s() LN%d, failed to read json, stb name not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); + + cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); + if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { + printf("ERROR: failed to read json, childtable_prefix not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE); + + cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null + if (autoCreateTbl + && autoCreateTbl->type == cJSON_String + && autoCreateTbl->valuestring != NULL) { + if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; + } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } + } else if (!autoCreateTbl) { + g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; + } else { + printf("ERROR: failed to read json, auto_create_table not found\n"); + goto PARSE_OVER; + } + + cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num"); + if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint; + } else if (!batchCreateTbl) { + g_Dbs.db[i].superTbls[j].batchCreateTableNum = 1000; + } else { + printf("ERROR: failed to read json, batch_create_tbl_num not found\n"); + goto PARSE_OVER; + } + + cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no + if (childTblExists + && childTblExists->type == cJSON_String + && childTblExists->valuestring != NULL) { + if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; + } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } + } else if (!childTblExists) { + g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; + } else { + errorPrint("%s() LN%d, failed to read json, child_table_exists not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); + if (!count || count->type != cJSON_Number || 0 >= count->valueint) { + errorPrint("%s() LN%d, failed to read json, childtable_count not found\n", + __func__, __LINE__); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; + + cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); + if (dataSource && dataSource->type == cJSON_String + && dataSource->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, + dataSource->valuestring, MAX_DB_NAME_SIZE); + } else if (!dataSource) { + tstrncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); + } else { + errorPrint("%s() LN%d, failed to read json, data_source not found\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful + if (insertMode && insertMode->type == cJSON_String + && insertMode->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, + insertMode->valuestring, MAX_DB_NAME_SIZE); + } else if (!insertMode) { + tstrncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE); + } else { + printf("ERROR: failed to read json, insert_mode not found\n"); + goto PARSE_OVER; + } + + cJSON* childTbl_limit = cJSON_GetObjectItem(stbInfo, "childtable_limit"); + if (childTbl_limit) { + if (childTbl_limit->type != cJSON_Number) { + printf("ERROR: failed to read json, childtable_limit\n"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblLimit = childTbl_limit->valueint; + } else { + g_Dbs.db[i].superTbls[j].childTblLimit = -1; // select ... limit -1 means all query result + } + + cJSON* childTbl_offset = cJSON_GetObjectItem(stbInfo, "childtable_offset"); + if (childTbl_offset) { + if (childTbl_offset->type != cJSON_Number || 0 > childTbl_offset->valueint) { + printf("ERROR: failed to read json, childtable_offset\n"); + goto PARSE_OVER; + } + g_Dbs.db[i].superTbls[j].childTblOffset = childTbl_offset->valueint; + } else { + g_Dbs.db[i].superTbls[j].childTblOffset = 0; + } + + cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); + if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, + ts->valuestring, MAX_DB_NAME_SIZE); + } else if (!ts) { + tstrncpy(g_Dbs.db[i].superTbls[j].startTimestamp, + "now", MAX_DB_NAME_SIZE); + } else { + printf("ERROR: failed to read json, start_timestamp not found\n"); + goto PARSE_OVER; + } + + cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); + if (timestampStep && timestampStep->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; + } else if (!timestampStep) { + g_Dbs.db[i].superTbls[j].timeStampStep = DEFAULT_TIMESTAMP_STEP; + } else { + printf("ERROR: failed to read json, timestamp_step not found\n"); + goto PARSE_OVER; + } + + cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size"); + if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint; + if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; + } + } else if (!sampleDataBufSize) { + g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; + } else { + printf("ERROR: failed to read json, sample_buf_size not found\n"); + goto PARSE_OVER; + } + + cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); + if (sampleFormat && sampleFormat->type + == cJSON_String && sampleFormat->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, + sampleFormat->valuestring, MAX_DB_NAME_SIZE); + } else if (!sampleFormat) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); + } else { + printf("ERROR: failed to read json, sample_format not found\n"); + goto PARSE_OVER; + } + + cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); + if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].sampleFile, + sampleFile->valuestring, MAX_FILE_NAME_LEN); + } else if (!sampleFile) { + memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); + } else { + printf("ERROR: failed to read json, sample_file not found\n"); + goto PARSE_OVER; + } + + cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); + if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { + tstrncpy(g_Dbs.db[i].superTbls[j].tagsFile, + tagsFile->valuestring, MAX_FILE_NAME_LEN); + if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + g_Dbs.db[i].superTbls[j].tagSource = 1; + } + } else if (!tagsFile) { + memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); + g_Dbs.db[i].superTbls[j].tagSource = 0; + } else { + printf("ERROR: failed to read json, tags_file not found\n"); + goto PARSE_OVER; + } + + cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); + if (maxSqlLen && maxSqlLen->type == cJSON_Number) { + int32_t len = maxSqlLen->valueint; + if (len > TSDB_MAX_ALLOWED_SQL_LEN) { + len = TSDB_MAX_ALLOWED_SQL_LEN; + } else if (len < TSDB_MAX_SQL_LEN) { + len = TSDB_MAX_SQL_LEN; + } + g_Dbs.db[i].superTbls[j].maxSqlLen = len; + } else if (!maxSqlLen) { + g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; + } else { + printf("ERROR: failed to read json, maxSqlLen not found\n"); + goto PARSE_OVER; + } + + cJSON *multiThreadWriteOneTbl = + cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes + if (multiThreadWriteOneTbl + && multiThreadWriteOneTbl->type == cJSON_String + && multiThreadWriteOneTbl->valuestring != NULL) { + if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; + } else { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } + } else if (!multiThreadWriteOneTbl) { + g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; + } else { + printf("ERROR: failed to read json, multiThreadWriteOneTbl not found\n"); + goto PARSE_OVER; + } + + cJSON* interlaceRows = cJSON_GetObjectItem(stbInfo, "interlace_rows"); + if (interlaceRows && interlaceRows->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].interlaceRows = interlaceRows->valueint; + // rows per table need be less than insert batch + if (g_Dbs.db[i].superTbls[j].interlaceRows > g_args.num_of_RPR) { + printf("NOTICE: db[%d].superTbl[%d]'s interlace rows value %d > num_of_records_per_request %d\n\n", + i, j, g_Dbs.db[i].superTbls[j].interlaceRows, g_args.num_of_RPR); + printf(" interlace rows value will be set to num_of_records_per_request %d\n\n", + g_args.num_of_RPR); + printf(" press Enter key to continue or Ctrl-C to stop."); + (void)getchar(); + g_Dbs.db[i].superTbls[j].interlaceRows = g_args.num_of_RPR; + } + } else if (!interlaceRows) { + g_Dbs.db[i].superTbls[j].interlaceRows = 0; // 0 means progressive mode, > 0 mean interlace mode. max value is less or equ num_of_records_per_req + } else { + errorPrint( + "%s() LN%d, failed to read json, interlace rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); + if (disorderRatio && disorderRatio->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; + } else if (!disorderRatio) { + g_Dbs.db[i].superTbls[j].disorderRatio = 0; + } else { + printf("ERROR: failed to read json, disorderRatio not found\n"); + goto PARSE_OVER; + } + + cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); + if (disorderRange && disorderRange->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; + } else if (!disorderRange) { + g_Dbs.db[i].superTbls[j].disorderRange = 1000; + } else { + printf("ERROR: failed to read json, disorderRange not found\n"); + goto PARSE_OVER; + } + + cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); + if (insertRows && insertRows->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; + } else if (!insertRows) { + g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; + } else { + errorPrint("%s() LN%d, failed to read json, insert_rows input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* insertInterval = cJSON_GetObjectItem(stbInfo, "insert_interval"); + if (insertInterval && insertInterval->type == cJSON_Number) { + g_Dbs.db[i].superTbls[j].insertInterval = insertInterval->valueint; + } else if (!insertInterval) { + verbosePrint("%s() LN%d: stable insert interval be overrided by global %d.\n", + __func__, __LINE__, g_args.insert_interval); + g_Dbs.db[i].superTbls[j].insertInterval = g_args.insert_interval; + } else { + errorPrint("%s() LN%d, failed to read json, insert_interval input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; + } + + int retVal = getColumnAndTagTypeFromInsertJsonFile( + stbInfo, &g_Dbs.db[i].superTbls[j]); + if (false == retVal) { + goto PARSE_OVER; + } + } + } + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getMetaFromQueryJsonFile(cJSON* root) { + bool ret = false; + + cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); + if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { + tstrncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); + } + + cJSON* host = cJSON_GetObjectItem(root, "host"); + if (host && host->type == cJSON_String && host->valuestring != NULL) { + tstrncpy(g_queryInfo.host, host->valuestring, MAX_HOSTNAME_SIZE); + } else if (!host) { + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + } else { + printf("ERROR: failed to read json, host not found\n"); + goto PARSE_OVER; + } + + cJSON* port = cJSON_GetObjectItem(root, "port"); + if (port && port->type == cJSON_Number) { + g_queryInfo.port = port->valueint; + } else if (!port) { + g_queryInfo.port = 6030; + } + + cJSON* user = cJSON_GetObjectItem(root, "user"); + if (user && user->type == cJSON_String && user->valuestring != NULL) { + tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); + } else if (!user) { + tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ; + } + + cJSON* password = cJSON_GetObjectItem(root, "password"); + if (password && password->type == cJSON_String && password->valuestring != NULL) { + tstrncpy(g_queryInfo.password, password->valuestring, MAX_PASSWORD_SIZE); + } else if (!password) { + tstrncpy(g_queryInfo.password, "taosdata", MAX_PASSWORD_SIZE);; + } + + cJSON *answerPrompt = cJSON_GetObjectItem(root, "confirm_parameter_prompt"); // yes, no, + if (answerPrompt && answerPrompt->type == cJSON_String + && answerPrompt->valuestring != NULL) { + if (0 == strncasecmp(answerPrompt->valuestring, "yes", 3)) { + g_args.answer_yes = false; + } else if (0 == strncasecmp(answerPrompt->valuestring, "no", 2)) { + g_args.answer_yes = true; + } else { + g_args.answer_yes = false; + } + } else if (!answerPrompt) { + g_args.answer_yes = false; + } else { + printf("ERROR: failed to read json, confirm_parameter_prompt not found\n"); + goto PARSE_OVER; + } + + cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); + if (gQueryTimes && gQueryTimes->type == cJSON_Number) { + g_args.query_times = gQueryTimes->valueint; + } else if (!gQueryTimes) { + g_args.query_times = 1; + } else { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* dbs = cJSON_GetObjectItem(root, "databases"); + if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { + tstrncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); + } else if (!dbs) { + printf("ERROR: failed to read json, databases not found\n"); + goto PARSE_OVER; + } + + cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); + if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { + tstrncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); + } else if (!queryMode) { + tstrncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); + } else { + printf("ERROR: failed to read json, query_mode not found\n"); + goto PARSE_OVER; + } + + // super_table_query + cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query"); + if (!specifiedQuery) { + g_queryInfo.specifiedQueryInfo.concurrent = 0; + g_queryInfo.specifiedQueryInfo.sqlCount = 0; + } else if (specifiedQuery->type != cJSON_Object) { + printf("ERROR: failed to read json, super_table_query not found\n"); + goto PARSE_OVER; + } else { + cJSON* rate = cJSON_GetObjectItem(specifiedQuery, "query_interval"); + if (rate && rate->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.rate = rate->valueint; + } else if (!rate) { + g_queryInfo.specifiedQueryInfo.rate = 0; + } + + cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times"); + if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.queryTimes = specifiedQueryTimes->valueint; + } else if (!specifiedQueryTimes) { + g_queryInfo.specifiedQueryInfo.queryTimes = g_args.query_times; + } else { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* concurrent = cJSON_GetObjectItem(specifiedQuery, "concurrent"); + if (concurrent && concurrent->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.concurrent = concurrent->valueint; + } else if (!concurrent) { + g_queryInfo.specifiedQueryInfo.concurrent = 1; + } + + cJSON* mode = cJSON_GetObjectItem(specifiedQuery, "mode"); + if (mode && mode->type == cJSON_String && mode->valuestring != NULL) { + if (0 == strcmp("sync", mode->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeMode = 0; + } else if (0 == strcmp("async", mode->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeMode = 1; + } else { + printf("ERROR: failed to read json, subscribe mod error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.specifiedQueryInfo.subscribeMode = 0; + } + + cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval"); + if (interval && interval->type == cJSON_Number) { + g_queryInfo.specifiedQueryInfo.subscribeInterval = interval->valueint; + } else if (!interval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.specifiedQueryInfo.subscribeInterval = 10000; + } + + cJSON* restart = cJSON_GetObjectItem(specifiedQuery, "restart"); + if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { + if (0 == strcmp("yes", restart->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + } else if (0 == strcmp("no", restart->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeRestart = 0; + } else { + printf("ERROR: failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.specifiedQueryInfo.subscribeRestart = 1; + } + + cJSON* keepProgress = cJSON_GetObjectItem(specifiedQuery, "keepProgress"); + if (keepProgress + && keepProgress->type == cJSON_String + && keepProgress->valuestring != NULL) { + if (0 == strcmp("yes", keepProgress->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", keepProgress->valuestring)) { + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; + } else { + printf("ERROR: failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.specifiedQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* superSqls = cJSON_GetObjectItem(specifiedQuery, "sqls"); + if (!superSqls) { + g_queryInfo.specifiedQueryInfo.sqlCount = 0; + } else if (superSqls->type != cJSON_Array) { + printf("ERROR: failed to read json, super sqls not found\n"); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(superSqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.specifiedQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(superSqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("ERROR: failed to read json, sql not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_queryInfo.specifiedQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { + tstrncpy(g_queryInfo.specifiedQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.specifiedQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + printf("ERROR: failed to read json, super query result file not found\n"); + goto PARSE_OVER; + } + } + } + } + + // sub_table_query + cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query"); + if (!superQuery) { + g_queryInfo.superQueryInfo.threadCnt = 0; + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (superQuery->type != cJSON_Object) { + printf("ERROR: failed to read json, sub_table_query not found\n"); + ret = true; + goto PARSE_OVER; + } else { + cJSON* subrate = cJSON_GetObjectItem(superQuery, "query_interval"); + if (subrate && subrate->type == cJSON_Number) { + g_queryInfo.superQueryInfo.rate = subrate->valueint; + } else if (!subrate) { + g_queryInfo.superQueryInfo.rate = 0; + } + + cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); + if (superQueryTimes && superQueryTimes->type == cJSON_Number) { + g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; + } else if (!superQueryTimes) { + g_queryInfo.superQueryInfo.queryTimes = g_args.query_times; + } else { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", __func__, __LINE__); + goto PARSE_OVER; + } + + cJSON* threads = cJSON_GetObjectItem(superQuery, "threads"); + if (threads && threads->type == cJSON_Number) { + g_queryInfo.superQueryInfo.threadCnt = threads->valueint; + } else if (!threads) { + g_queryInfo.superQueryInfo.threadCnt = 1; + } + + //cJSON* subTblCnt = cJSON_GetObjectItem(superQuery, "childtable_count"); + //if (subTblCnt && subTblCnt->type == cJSON_Number) { + // g_queryInfo.superQueryInfo.childTblCount = subTblCnt->valueint; + //} else if (!subTblCnt) { + // g_queryInfo.superQueryInfo.childTblCount = 0; + //} + + cJSON* stblname = cJSON_GetObjectItem(superQuery, "stblname"); + if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { + tstrncpy(g_queryInfo.superQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); + } else { + printf("ERROR: failed to read json, super table name not found\n"); + goto PARSE_OVER; + } + + cJSON* submode = cJSON_GetObjectItem(superQuery, "mode"); + if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { + if (0 == strcmp("sync", submode->valuestring)) { + g_queryInfo.superQueryInfo.subscribeMode = 0; + } else if (0 == strcmp("async", submode->valuestring)) { + g_queryInfo.superQueryInfo.subscribeMode = 1; + } else { + printf("ERROR: failed to read json, subscribe mod error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeMode = 0; + } + + cJSON* subinterval = cJSON_GetObjectItem(superQuery, "interval"); + if (subinterval && subinterval->type == cJSON_Number) { + g_queryInfo.superQueryInfo.subscribeInterval = subinterval->valueint; + } else if (!subinterval) { + //printf("failed to read json, subscribe interval no found\n"); + //goto PARSE_OVER; + g_queryInfo.superQueryInfo.subscribeInterval = 10000; + } + + cJSON* subrestart = cJSON_GetObjectItem(superQuery, "restart"); + if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { + if (0 == strcmp("yes", subrestart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = 1; + } else if (0 == strcmp("no", subrestart->valuestring)) { + g_queryInfo.superQueryInfo.subscribeRestart = 0; + } else { + printf("ERROR: failed to read json, subscribe restart error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeRestart = 1; + } + + cJSON* subkeepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); + if (subkeepProgress && + subkeepProgress->type == cJSON_String + && subkeepProgress->valuestring != NULL) { + if (0 == strcmp("yes", subkeepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; + } else if (0 == strcmp("no", subkeepProgress->valuestring)) { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } else { + printf("ERROR: failed to read json, subscribe keepProgress error\n"); + goto PARSE_OVER; + } + } else { + g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; + } + + // sqls + cJSON* subsqls = cJSON_GetObjectItem(superQuery, "sqls"); + if (!subsqls) { + g_queryInfo.superQueryInfo.sqlCount = 0; + } else if (subsqls->type != cJSON_Array) { + printf("ERROR: failed to read json, super sqls not found\n"); + goto PARSE_OVER; + } else { + int superSqlSize = cJSON_GetArraySize(subsqls); + if (superSqlSize > MAX_QUERY_SQL_COUNT) { + printf("ERROR: failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); + goto PARSE_OVER; + } + + g_queryInfo.superQueryInfo.sqlCount = superSqlSize; + for (int j = 0; j < superSqlSize; ++j) { + cJSON* sql = cJSON_GetArrayItem(subsqls, j); + if (sql == NULL) continue; + + cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); + if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { + printf("ERROR: failed to read json, sql not found\n"); + goto PARSE_OVER; + } + tstrncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); + + cJSON *result = cJSON_GetObjectItem(sql, "result"); + if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ + tstrncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); + } else if (NULL == result) { + memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); + } else { + printf("ERROR: failed to read json, sub query result file not found\n"); + goto PARSE_OVER; + } + } + } + } + + ret = true; + +PARSE_OVER: + //free(content); + //cJSON_Delete(root); + //fclose(fp); + return ret; +} + +static bool getInfoFromJsonFile(char* file) { + debugPrint("%s %d %s\n", __func__, __LINE__, file); + + FILE *fp = fopen(file, "r"); + if (!fp) { + printf("failed to read %s, reason:%s\n", file, strerror(errno)); + return false; + } + + bool ret = false; + int maxLen = 6400000; + char *content = calloc(1, maxLen + 1); + int len = fread(content, 1, maxLen, fp); + if (len <= 0) { + free(content); + fclose(fp); + printf("failed to read %s, content is null", file); + return false; + } + + content[len] = 0; + cJSON* root = cJSON_Parse(content); + if (root == NULL) { + printf("ERROR: failed to cjson parse %s, invalid json format\n", file); + goto PARSE_OVER; + } + + cJSON* filetype = cJSON_GetObjectItem(root, "filetype"); + if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) { + if (0 == strcasecmp("insert", filetype->valuestring)) { + g_args.test_mode = INSERT_TEST; + } else if (0 == strcasecmp("query", filetype->valuestring)) { + g_args.test_mode = QUERY_TEST; + } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { + g_args.test_mode = SUBSCRIBE_TEST; + } else { + printf("ERROR: failed to read json, filetype not support\n"); + goto PARSE_OVER; + } + } else if (!filetype) { + g_args.test_mode = INSERT_TEST; + } else { + printf("ERROR: failed to read json, filetype not found\n"); + goto PARSE_OVER; + } + + if (INSERT_TEST == g_args.test_mode) { + ret = getMetaFromInsertJsonFile(root); + } else if ((QUERY_TEST == g_args.test_mode) + || (SUBSCRIBE_TEST == g_args.test_mode)) { + ret = getMetaFromQueryJsonFile(root); + } else { + errorPrint("%s() LN%d, input json file type error! please input correct file type: insert or query or subscribe\n", + __func__, __LINE__); + goto PARSE_OVER; + } + +PARSE_OVER: + free(content); + cJSON_Delete(root); + fclose(fp); + return ret; +} + +static void prepareSampleData() { + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { + (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]); + } + } + } +} + +static void postFreeResource() { + tmfclose(g_fpOfInsertResult); + for (int i = 0; i < g_Dbs.dbCount; i++) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if (0 != g_Dbs.db[i].superTbls[j].colsOfCreateChildTable) { + free(g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) { + free(g_Dbs.db[i].superTbls[j].sampleDataBuf); + g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) { + free(g_Dbs.db[i].superTbls[j].tagDataBuf); + g_Dbs.db[i].superTbls[j].tagDataBuf = NULL; + } + if (0 != g_Dbs.db[i].superTbls[j].childTblName) { + free(g_Dbs.db[i].superTbls[j].childTblName); + g_Dbs.db[i].superTbls[j].childTblName = NULL; + } + } + } +} + +static int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, + SSuperTable* superTblInfo, int* sampleUsePos) { + if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) { +/* int ret = readSampleFromCsvFileToMem(superTblInfo); + if (0 != ret) { + tmfree(superTblInfo->sampleDataBuf); + superTblInfo->sampleDataBuf = NULL; + return -1; + } +*/ + *sampleUsePos = 0; + } + + int dataLen = 0; + + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + "(%" PRId64 ", ", timestamp); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + "%s", superTblInfo->sampleDataBuf + superTblInfo->lenOfOneRow * (*sampleUsePos)); + dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + + (*sampleUsePos)++; + + return dataLen; +} + +static int generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) { + int dataLen = 0; + char *pstr = recBuf; + int maxLen = MAX_DATA_SIZE; + + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + + for (int i = 0; i < stbInfo->columnCount; i++) { + if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6)) + || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) { + if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { + errorPrint( "binary or nchar length overflow, max size:%u\n", + (uint32_t)TSDB_MAX_BINARY_LEN); + return -1; + } + + char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1); + if (NULL == buf) { + errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen); + return -1; + } + rand_string(buf, stbInfo->columns[i].dataLen); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\', ", buf); + tmfree(buf); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "int", 3)) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "%d, ", rand_int()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "bigint", 6)) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "%"PRId64", ", rand_bigint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "float", 5)) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "%f, ", rand_float()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "double", 6)) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, + "%f, ", rand_double()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, + "smallint", 8)) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_bool()); + } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) { + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + } else { + errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType); + return -1; + } + } + + dataLen -= 2; + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")"); + + verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); + + return strlen(recBuf); +} + +static int32_t generateData(char *recBuf, char **data_type, + int num_of_cols, int64_t timestamp, int lenOfBinary) { + memset(recBuf, 0, MAX_DATA_SIZE); + char *pstr = recBuf; + pstr += sprintf(pstr, "(%" PRId64, timestamp); + int c = 0; + + for (; c < MAX_NUM_DATATYPE; c++) { + if (data_type[c] == NULL) { + break; + } + } + + if (0 == c) { + perror("data type error!"); exit(-1); } - /* Execute regular expression */ - int reti = regexec(®ex, s, 0, NULL, 0); - if (!reti) { - regfree(®ex); - return 1; - } else if (reti == REG_NOMATCH) { - regfree(®ex); - return 0; + for (int i = 0; i < num_of_cols; i++) { + if (strcasecmp(data_type[i % c], "tinyint") == 0) { + pstr += sprintf(pstr, ", %d", rand_tinyint() ); + } else if (strcasecmp(data_type[i % c], "smallint") == 0) { + pstr += sprintf(pstr, ", %d", rand_smallint()); + } else if (strcasecmp(data_type[i % c], "int") == 0) { + pstr += sprintf(pstr, ", %d", rand_int()); + } else if (strcasecmp(data_type[i % c], "bigint") == 0) { + pstr += sprintf(pstr, ", %" PRId64, rand_bigint()); + } else if (strcasecmp(data_type[i % c], "float") == 0) { + pstr += sprintf(pstr, ", %10.4f", rand_float()); + } else if (strcasecmp(data_type[i % c], "double") == 0) { + double t = rand_double(); + pstr += sprintf(pstr, ", %20.8f", t); + } else if (strcasecmp(data_type[i % c], "bool") == 0) { + bool b = taosRandom() & 1; + pstr += sprintf(pstr, ", %s", b ? "true" : "false"); + } else if (strcasecmp(data_type[i % c], "binary") == 0) { + char *s = malloc(lenOfBinary); + rand_string(s, lenOfBinary); + pstr += sprintf(pstr, ", \"%s\"", s); + free(s); + }else if (strcasecmp(data_type[i % c], "nchar") == 0) { + char *s = malloc(lenOfBinary); + rand_string(s, lenOfBinary); + pstr += sprintf(pstr, ", \"%s\"", s); + free(s); + } + + if (strlen(recBuf) > MAX_DATA_SIZE) { + perror("column length too long, abort"); + exit(-1); + } + } + + pstr += sprintf(pstr, ")"); + + return (int32_t)strlen(recBuf); +} + +static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { + char* sampleDataBuf = NULL; + + sampleDataBuf = calloc( + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); + if (sampleDataBuf == NULL) { + errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n", + __func__, __LINE__, + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, + strerror(errno)); + return -1; + } + + superTblInfo->sampleDataBuf = sampleDataBuf; + int ret = readSampleFromCsvFileToMem(superTblInfo); + + if (0 != ret) { + errorPrint("%s() LN%d, read sample from csv file failed.\n", __func__, __LINE__); + tmfree(sampleDataBuf); + superTblInfo->sampleDataBuf = NULL; + return -1; + } + + return 0; +} + +static int execInsert(threadInfo *pThreadInfo, char *buffer, int k) +{ + int affectedRows; + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + + verbosePrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, + __func__, __LINE__, buffer); + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { + affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false); + } else { + if (0 != postProceSql(g_Dbs.host, g_Dbs.port, buffer)) { + affectedRows = -1; + printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID); + } else { + affectedRows = k; + } + } + } else { + affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false); + } + + return affectedRows; +} + +static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq) +{ + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + if (superTblInfo) { + if ((superTblInfo->childTblOffset >= 0) + && (superTblInfo->childTblLimit > 0)) { + snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", + superTblInfo->childTblName + + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); + } else { + + verbosePrint("[%d] %s() LN%d: from=%d count=%d seq=%d\n", + pThreadInfo->threadID, __func__, __LINE__, + pThreadInfo->start_table_from, + pThreadInfo->ntables, tableSeq); + snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", + superTblInfo->childTblName + tableSeq * TSDB_TABLE_NAME_LEN); + } + } else { + snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s%d", + g_args.tb_prefix, tableSeq); + } +} + +static int generateDataTail(char *tableName, int32_t tableSeq, + threadInfo* pThreadInfo, SSuperTable* superTblInfo, + int batch, char* buffer, int remainderBufLen, int64_t insertRows, + int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) { + int len = 0; + int ncols_per_record = 1; // count first col ts + + if (superTblInfo == NULL) { + int datatypeSeq = 0; + while(g_args.datatype[datatypeSeq]) { + datatypeSeq ++; + ncols_per_record ++; + } + } + + verbosePrint("%s() LN%d batch=%d\n", __func__, __LINE__, batch); + + int k = 0; + for (k = 0; k < batch;) { + char data[MAX_DATA_SIZE]; + int retLen = 0; + + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample"))) { + retLen = getRowDataFromSample( + data, + remainderBufLen, + startTime + superTblInfo->timeStampStep * k, + superTblInfo, + pSamplePos); + } else if (0 == strncasecmp(superTblInfo->dataSource, + "rand", strlen("rand"))) { + int rand_num = rand_tinyint() % 100; + if (0 != superTblInfo->disorderRatio + && rand_num < superTblInfo->disorderRatio) { + int64_t d = startTime + + superTblInfo->timeStampStep * k + - taosRandom() % superTblInfo->disorderRange; + retLen = generateRowData( + data, + d, + superTblInfo); + } else { + retLen = generateRowData( + data, + startTime + superTblInfo->timeStampStep * k, + superTblInfo); + } + } + + if (retLen > remainderBufLen) { + break; + } + + buffer += snprintf(buffer, retLen + 1, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; + } else { + int rand_num = taosRandom() % 100; + char **data_type = g_args.datatype; + int lenOfBinary = g_args.len_of_binary; + + if ((g_args.disorderRatio != 0) + && (rand_num < g_args.disorderRange)) { + + int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k + - taosRandom() % 1000000 + rand_num; + retLen = generateData(data, data_type, + ncols_per_record, d, lenOfBinary); + } else { + retLen = generateData(data, data_type, + ncols_per_record, + startTime + DEFAULT_TIMESTAMP_STEP * k, + lenOfBinary); + } + + if (len > remainderBufLen) + break; + + buffer += sprintf(buffer, " %s", data); + k++; + len += retLen; + remainderBufLen -= retLen; + } + + verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n", + __func__, __LINE__, len, k, buffer); + + startFrom ++; + + if (startFrom >= insertRows) { + break; + } + } + + *dataLen = len; + return k; +} + +static int generateSQLHead(char *tableName, int32_t tableSeq, + threadInfo* pThreadInfo, SSuperTable* superTblInfo, char *buffer) +{ + int len; + if (superTblInfo) { + if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { + char* tagsValBuf = NULL; + if (0 == superTblInfo->tagSource) { + tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq); + } else { + tagsValBuf = getTagValueFromTagSample( + superTblInfo, + tableSeq % superTblInfo->tagSampleCount); + } + if (NULL == tagsValBuf) { + errorPrint("%s() LN%d, tag buf failed to allocate memory\n", __func__, __LINE__); + return -1; + } + + len = snprintf(buffer, + superTblInfo->maxSqlLen, + "insert into %s.%s using %s.%s tags %s values", + pThreadInfo->db_name, + tableName, + pThreadInfo->db_name, + superTblInfo->sTblName, + tagsValBuf); + tmfree(tagsValBuf); + } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { + len = snprintf(buffer, + superTblInfo->maxSqlLen, + "insert into %s.%s values", + pThreadInfo->db_name, + tableName); + } else { + len = snprintf(buffer, + superTblInfo->maxSqlLen, + "insert into %s.%s values", + pThreadInfo->db_name, + tableName); + } } else { - regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); - printf("Regex match failed: %s\n", msgbuf); - regfree(®ex); - exit(-1); + len = snprintf(buffer, + g_args.max_sql_len, + "insert into %s.%s values", + pThreadInfo->db_name, + tableName); } - return 0; + return len; } -static int isCommentLine(char *line) { - if (line == NULL) return 1; +static int generateProgressiveDataBuffer(char *pTblName, + int32_t tableSeq, + threadInfo *pThreadInfo, char *buffer, + int64_t insertRows, + int64_t startFrom, int64_t startTime, int *pSamplePos) +{ + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; - return regexMatch(line, "^\\s*#.*", REG_EXTENDED); + int ncols_per_record = 1; // count first col ts + + if (superTblInfo == NULL) { + int datatypeSeq = 0; + while(g_args.datatype[datatypeSeq]) { + datatypeSeq ++; + ncols_per_record ++; + } + } + + assert(buffer != NULL); + + int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int remainderBufLen = maxSqlLen; + + memset(buffer, 0, maxSqlLen); + + char *pstr = buffer; + + int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo, + buffer); + pstr += headLen; + remainderBufLen -= headLen; + + int k; + int dataLen; + k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo, + g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom, + startTime, + pSamplePos, &dataLen); + + return k; } -void querySqlFile(TAOS* taos, char* sqlFile) -{ - FILE *fp = fopen(sqlFile, "r"); - if (fp == NULL) { - printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); - exit(-1); +static void* syncWriteInterlace(threadInfo *pThreadInfo) { + debugPrint("[%d] %s() LN%d: ### interlace write\n", + pThreadInfo->threadID, __func__, __LINE__); + + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; + + int insertMode; + + if (interlaceRows > 0) { + insertMode = INTERLACE_INSERT_MODE; + } else { + insertMode = PROGRESSIVE_INSERT_MODE; } - - int read_len = 0; - char * cmd = calloc(1, MAX_SQL_SIZE); - size_t cmd_len = 0; - char * line = NULL; - size_t line_len = 0; - double t = getCurrentTime(); - - while ((read_len = tgetline(&line, &line_len, fp)) != -1) { - if (read_len >= MAX_SQL_SIZE) continue; - line[--read_len] = '\0'; + // TODO: prompt tbl count multple interlace rows and batch + // - if (read_len == 0 || isCommentLine(line)) { // line starts with # - continue; + char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1); + if (NULL == buffer) { + errorPrint( "Failed to alloc %d Bytes, reason:%s\n", + superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, + strerror(errno)); + return NULL; + } + + + char tableName[TSDB_TABLE_NAME_LEN]; + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + int insert_interval = + superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; + uint64_t st = 0; + uint64_t et = 0xffffffff; + + int64_t lastPrintTime = taosGetTimestampMs(); + int64_t startTs = taosGetTimestampUs(); + int64_t endTs; + + int tableSeq = pThreadInfo->start_table_from; + + debugPrint("[%d] %s() LN%d: start_table_from=%d ntables=%d insertRows=%"PRId64"\n", + pThreadInfo->threadID, __func__, __LINE__, pThreadInfo->start_table_from, + pThreadInfo->ntables, insertRows); + + int64_t startTime = pThreadInfo->start_time; + + int batchPerTblTimes; + int batchPerTbl; + + assert(pThreadInfo->ntables > 0); + + if (interlaceRows > g_args.num_of_RPR) + interlaceRows = g_args.num_of_RPR; + + batchPerTbl = interlaceRows; + if ((interlaceRows > 0) && (pThreadInfo->ntables > 1)) { + batchPerTblTimes = + (g_args.num_of_RPR / (interlaceRows * pThreadInfo->ntables)) + 1; + } else { + batchPerTblTimes = 1; + } + + int generatedRecPerTbl = 0; + bool flagSleep = true; + int sleepTimeTotal = 0; + + int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int remainderBufLen; + + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { + if ((flagSleep) && (insert_interval)) { + st = taosGetTimestampUs(); + flagSleep = false; } + // generate data + memset(buffer, 0, maxSqlLen); + remainderBufLen = maxSqlLen; + + char *pstr = buffer; + int recOfBatch = 0; + + for (int i = 0; i < batchPerTblTimes; i ++) { + getTableName(tableName, pThreadInfo, tableSeq); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + return NULL; + exit(-1); + } - if (line[read_len - 1] == '\\') { - line[read_len - 1] = ' '; - memcpy(cmd + cmd_len, line, read_len); - cmd_len += read_len; - continue; + int headLen; + if (i == 0) { + headLen = generateSQLHead(tableName, tableSeq, pThreadInfo, + superTblInfo, pstr); + } else { + headLen = snprintf(pstr, TSDB_TABLE_NAME_LEN, "%s.%s values", + pThreadInfo->db_name, + tableName); + } + + // generate data buffer + verbosePrint("[%d] %s() LN%d i=%d buffer:\n%s\n", + pThreadInfo->threadID, __func__, __LINE__, i, buffer); + + pstr += headLen; + remainderBufLen -= headLen; + + int dataLen = 0; + + verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", + pThreadInfo->threadID, __func__, __LINE__, + i, batchPerTblTimes, batchPerTbl); + + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + startTime = taosGetTimestamp(pThreadInfo->time_precision); + } + } else { + startTime = 1500000000000; + } + int generated = generateDataTail( + tableName, tableSeq, pThreadInfo, superTblInfo, + batchPerTbl, pstr, remainderBufLen, insertRows, 0, + startTime, + &(pThreadInfo->samplePos), &dataLen); + + if (generated < 0) { + debugPrint("[%d] %s() LN%d, generated data is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + goto free_and_statistics_interlace; + } + pstr += dataLen; + remainderBufLen -= dataLen; + + recOfBatch += batchPerTbl; +// startTime += batchPerTbl * superTblInfo->timeStampStep; + pThreadInfo->totalInsertRows += batchPerTbl; + verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", + pThreadInfo->threadID, __func__, __LINE__, + batchPerTbl, recOfBatch); + + tableSeq ++; + if (insertMode == INTERLACE_INSERT_MODE) { + if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { + // turn to first table + tableSeq = pThreadInfo->start_table_from; + generatedRecPerTbl += batchPerTbl; + + startTime = pThreadInfo->start_time + + generatedRecPerTbl * superTblInfo->timeStampStep; + + flagSleep = true; + if (generatedRecPerTbl >= insertRows) + break; + + if (pThreadInfo->ntables * batchPerTbl < g_args.num_of_RPR) + break; + } + } + + int remainRows = insertRows - generatedRecPerTbl; + if ((remainRows > 0) && (batchPerTbl > remainRows)) + batchPerTbl = remainRows; + + verbosePrint("[%d] %s() LN%d generatedRecPerTbl=%d insertRows=%"PRId64"\n", + pThreadInfo->threadID, __func__, __LINE__, + generatedRecPerTbl, insertRows); + + if ((g_args.num_of_RPR - recOfBatch) < batchPerTbl) + break; } - memcpy(cmd + cmd_len, line, read_len); - selectSql(taos, cmd); - memset(cmd, 0, MAX_SQL_SIZE); - cmd_len = 0; + verbosePrint("[%d] %s() LN%d recOfBatch=%d totalInsertRows=%"PRId64"\n", + pThreadInfo->threadID, __func__, __LINE__, recOfBatch, + pThreadInfo->totalInsertRows); + verbosePrint("[%d] %s() LN%d, buffer=%s\n", + pThreadInfo->threadID, __func__, __LINE__, buffer); + + startTs = taosGetTimestampUs(); + + int affectedRows = execInsert(pThreadInfo, buffer, recOfBatch); + + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + performancePrint("%s() LN%d, insert execution time is %10.6fms\n", + __func__, __LINE__, delay/1000.0); + + if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; + if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; + pThreadInfo->cntDelay++; + pThreadInfo->totalDelay += delay; + + verbosePrint("[%d] %s() LN%d affectedRows=%d\n", pThreadInfo->threadID, + __func__, __LINE__, affectedRows); + if ((affectedRows < 0) || (recOfBatch != affectedRows)) { + errorPrint("[%d] %s() LN%d execInsert insert %d, affected rows: %d\n%s\n", + pThreadInfo->threadID, __func__, __LINE__, + recOfBatch, affectedRows, buffer); + goto free_and_statistics_interlace; + } + + pThreadInfo->totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + lastPrintTime = currentPrintTime; + } + + if ((insert_interval) && flagSleep) { + et = taosGetTimestampUs(); + + if (insert_interval > ((et - st)/1000) ) { + int sleepTime = insert_interval - (et -st)/1000; + performancePrint("%s() LN%d sleep: %d ms for insert interval\n", + __func__, __LINE__, sleepTime); + taosMsleep(sleepTime); // ms + sleepTimeTotal += insert_interval; + } + } } - t = getCurrentTime() - t; - printf("run %s took %.6f second(s)\n\n", sqlFile, t); +free_and_statistics_interlace: + tmfree(buffer); - free(cmd); - if (line) free(line); - fclose(fp); - return; + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + return NULL; } -void * createTable(void *sarg) -{ - char command[BUFFER_SIZE] = "\0"; - - info *winfo = (info *)sarg; +// sync insertion +/* + 1 thread: 100 tables * 2000 rows/s + 1 thread: 10 tables * 20000 rows/s + 6 thread: 300 tables * 2000 rows/s + + 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s +*/ +static void* syncWriteProgressive(threadInfo *pThreadInfo) { + debugPrint("%s() LN%d: ### progressive write\n", __func__, __LINE__); + + SSuperTable* superTblInfo = pThreadInfo->superTblInfo; + + char* buffer = calloc(superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, 1); + if (NULL == buffer) { + errorPrint( "Failed to alloc %d Bytes, reason:%s\n", + superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len, + strerror(errno)); + return NULL; + } + + int64_t lastPrintTime = taosGetTimestampMs(); + int64_t startTs = taosGetTimestampUs(); + int64_t endTs; + + int timeStampStep = + superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; +/* int insert_interval = + superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; + uint64_t st = 0; + uint64_t et = 0xffffffff; + */ + + pThreadInfo->totalInsertRows = 0; + pThreadInfo->totalAffectedRows = 0; + + pThreadInfo->samplePos = 0; + + for (uint32_t tableSeq = + pThreadInfo->start_table_from; tableSeq <= pThreadInfo->end_table_to; + tableSeq ++) { + int64_t start_time = pThreadInfo->start_time; + + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; + verbosePrint("%s() LN%d insertRows=%"PRId64"\n", __func__, __LINE__, insertRows); + + for (int64_t i = 0; i < insertRows;) { + /* + if (insert_interval) { + st = taosGetTimestampUs(); + } + */ + + char tableName[TSDB_TABLE_NAME_LEN]; + getTableName(tableName, pThreadInfo, tableSeq); + verbosePrint("%s() LN%d: tid=%d seq=%d tableName=%s\n", + __func__, __LINE__, + pThreadInfo->threadID, tableSeq, tableName); + + int generated = generateProgressiveDataBuffer( + tableName, tableSeq, pThreadInfo, buffer, insertRows, + i, start_time, + &(pThreadInfo->samplePos)); + if (generated > 0) + i += generated; + else + goto free_and_statistics_2; + + start_time += generated * timeStampStep; + pThreadInfo->totalInsertRows += generated; + + startTs = taosGetTimestampUs(); + + int affectedRows = execInsert(pThreadInfo, buffer, generated); + + endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + performancePrint("%s() LN%d, insert execution time is %10.6fms\n", + __func__, __LINE__, delay/1000.0); + + if (delay > pThreadInfo->maxDelay) pThreadInfo->maxDelay = delay; + if (delay < pThreadInfo->minDelay) pThreadInfo->minDelay = delay; + pThreadInfo->cntDelay++; + pThreadInfo->totalDelay += delay; - if (!winfo->use_metric) { - /* Create all the tables; */ - printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); - for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d (ts timestamp%s);", winfo->db_name, winfo->tb_prefix, i, winfo->cols); - queryDB(winfo->taos, command); + if (affectedRows < 0) + goto free_and_statistics_2; + + pThreadInfo->totalAffectedRows += affectedRows; + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 30*1000) { + printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + lastPrintTime = currentPrintTime; + } + + if (i >= insertRows) + break; +/* + if (insert_interval) { + et = taosGetTimestampUs(); + + if (insert_interval > ((et - st)/1000) ) { + int sleep_time = insert_interval - (et -st)/1000; + performancePrint("%s() LN%d sleep: %d ms for insert interval\n", + __func__, __LINE__, sleep_time); + taosMsleep(sleep_time); // ms + } + } + */ + } // num_of_DPT + + if ((tableSeq == pThreadInfo->ntables - 1) && superTblInfo && + (0 == strncasecmp( + superTblInfo->dataSource, "sample", strlen("sample")))) { + printf("%s() LN%d samplePos=%d\n", + __func__, __LINE__, pThreadInfo->samplePos); } + } // tableSeq + +free_and_statistics_2: + tmfree(buffer); + + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, + pThreadInfo->totalAffectedRows); + return NULL; +} + +static void* syncWrite(void *sarg) { + + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; + + if (interlaceRows > 0) { + // interlace mode + return syncWriteInterlace(winfo); } else { - /* Create all the tables; */ - printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); - for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - int j; - if (i % 10 == 0) { - j = 10; - } else { - j = i % 10; - } - if (j % 2 == 0) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.meters tags (%d,\"%s\");", winfo->db_name, winfo->tb_prefix, i, winfo->db_name, j, "shanghai"); + // progressive mode + return syncWriteProgressive(winfo); + } +} + +static void callBack(void *param, TAOS_RES *res, int code) { + threadInfo* winfo = (threadInfo*)param; + SSuperTable* superTblInfo = winfo->superTblInfo; + + int insert_interval = + superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; + if (insert_interval) { + winfo->et = taosGetTimestampUs(); + if (((winfo->et - winfo->st)/1000) < insert_interval) { + taosMsleep(insert_interval - (winfo->et - winfo->st)/1000); // ms + } + } + + char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); + char data[MAX_DATA_SIZE]; + char *pstr = buffer; + pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, + winfo->start_table_from); +// if (winfo->counter >= winfo->superTblInfo->insertRows) { + if (winfo->counter >= g_args.num_of_RPR) { + winfo->start_table_from++; + winfo->counter = 0; + } + if (winfo->start_table_from > winfo->end_table_to) { + tsem_post(&winfo->lock_sem); + free(buffer); + taos_free_result(res); + return; + } + + for (int i = 0; i < g_args.num_of_RPR; i++) { + int rand_num = taosRandom() % 100; + if (0 != winfo->superTblInfo->disorderRatio + && rand_num < winfo->superTblInfo->disorderRatio) { + int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num; + generateRowData(data, d, winfo->superTblInfo); } else { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.meters tags (%d,\"%s\");", winfo->db_name, winfo->tb_prefix, i, winfo->db_name, j, "beijing"); + generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo); } - queryDB(winfo->taos, command); + pstr += sprintf(pstr, "%s", data); + winfo->counter++; + + if (winfo->counter >= winfo->superTblInfo->insertRows) { + break; } } + if (insert_interval) { + winfo->st = taosGetTimestampUs(); + } + taos_query_a(winfo->taos, buffer, callBack, winfo); + free(buffer); + + taos_free_result(res); +} + +static void *asyncWrite(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + SSuperTable* superTblInfo = winfo->superTblInfo; + + winfo->st = 0; + winfo->et = 0; + winfo->lastTs = winfo->start_time; + + int insert_interval = + superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; + if (insert_interval) { + winfo->st = taosGetTimestampUs(); + } + taos_query_a(winfo->taos, "show databases", callBack, winfo); + + tsem_wait(&(winfo->lock_sem)); + return NULL; } -void multiThreadCreateTable(char* cols, bool use_metric, int threads, int ntables, char* db_name, char* tb_prefix, char *ip_addr, uint16_t port, char *user, char *pass) { - double ts = getCurrentTime(); - printf("create table......\n"); +static void startMultiThreadInsertData(int threads, char* db_name, + char* precision,SSuperTable* superTblInfo) { + pthread_t *pids = malloc(threads * sizeof(pthread_t)); - info *infos = malloc(threads * sizeof(info)); + assert(pids != NULL); + + threadInfo *infos = malloc(threads * sizeof(threadInfo)); + assert(infos != NULL); + + memset(pids, 0, threads * sizeof(pthread_t)); + memset(infos, 0, threads * sizeof(threadInfo)); + + //TAOS* taos; + //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { + // taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); + // if (NULL == taos) { + // printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); + // exit(-1); + // } + //} + + int32_t timePrec = TSDB_TIME_PRECISION_MILLI; + if (0 != precision[0]) { + if (0 == strncasecmp(precision, "ms", 2)) { + timePrec = TSDB_TIME_PRECISION_MILLI; + } else if (0 == strncasecmp(precision, "us", 2)) { + timePrec = TSDB_TIME_PRECISION_MICRO; + } else { + errorPrint("Not support precision: %s\n", precision); + exit(-1); + } + } + + int64_t start_time; + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { + start_time = taosGetTimestamp(timePrec); + } else { + if (TSDB_CODE_SUCCESS != taosParseTime( + superTblInfo->startTimestamp, + &start_time, + strlen(superTblInfo->startTimestamp), + timePrec, 0)) { + ERROR_EXIT("failed to parse time!\n"); + } + } + } else { + start_time = 1500000000000; + } + + double start = getCurrentTime(); + + // read sample data from file first + if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample")))) { + if (0 != prepareSampleDataForSTable(superTblInfo)) { + errorPrint("%s() LN%d, prepare sample data for stable failed!\n", + __func__, __LINE__); + exit(-1); + } + } + + // read sample data from file first + if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, + "sample", strlen("sample")))) { + if (0 != prepareSampleDataForSTable(superTblInfo)) { + errorPrint("%s() LN%d, prepare sample data for stable failed!\n", + __func__, __LINE__); + exit(-1); + } + } + + TAOS* taos = taos_connect( + g_Dbs.host, g_Dbs.user, + g_Dbs.password, db_name, g_Dbs.port); + if (NULL == taos) { + errorPrint("%s() LN%d, connect to server fail , reason: %s\n", + __func__, __LINE__, taos_errstr(NULL)); + exit(-1); + } + + int ntables = 0; + int startFrom; + + if (superTblInfo) { + int limit, offset; + + if (superTblInfo->childTblOffset >= superTblInfo->childTblCount) { + printf("WARNING: specified offset >= child table count! \n"); + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } + } + + if (superTblInfo->childTblOffset >= 0) { + if (superTblInfo->childTblLimit <= 0) { + superTblInfo->childTblLimit = + superTblInfo->childTblCount - superTblInfo->childTblOffset; + } + + offset = superTblInfo->childTblOffset; + limit = superTblInfo->childTblLimit; + } else { + limit = superTblInfo->childTblCount; + offset = 0; + } + + ntables = limit; + startFrom = offset; + + superTblInfo->childTblName = (char*)calloc(1, + limit * TSDB_TABLE_NAME_LEN); + if (superTblInfo->childTblName == NULL) { + errorPrint("%s() LN%d, alloc memory failed!\n", __func__, __LINE__); + taos_close(taos); + exit(-1); + } + + int childTblCount; + getChildNameOfSuperTableWithLimitAndOffset( + taos, + db_name, superTblInfo->sTblName, + &superTblInfo->childTblName, &childTblCount, + limit, + offset); + } else { + ntables = g_args.num_of_tables; + startFrom = 0; + } + + taos_close(taos); int a = ntables / threads; if (a < 1) { @@ -1064,59 +5198,159 @@ void multiThreadCreateTable(char* cols, bool use_metric, int threads, int ntable a = 1; } - int b = 0; - if (threads != 0) - b = ntables % threads; - int last = 0; - for (int i = 0; i < threads; i++) { - info *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - tstrncpy(t_info->tb_prefix, tb_prefix, MAX_TB_NAME_SIZE); - t_info->taos = taos_connect(ip_addr, user, pass, db_name, port); - t_info->start_table_id = last; - t_info->end_table_id = i < b ? last + a : last + a - 1; - last = t_info->end_table_id + 1; - t_info->use_metric = use_metric; - t_info->cols = cols; - pthread_create(pids + i, NULL, createTable, t_info); + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); + t_info->time_precision = timePrec; + t_info->superTblInfo = superTblInfo; + + t_info->start_time = start_time; + t_info->minDelay = INT16_MAX; + + if ((NULL == superTblInfo) || + (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5))) { + //t_info->taos = taos; + t_info->taos = taos_connect( + g_Dbs.host, g_Dbs.user, + g_Dbs.password, db_name, g_Dbs.port); + if (NULL == t_info->taos) { + errorPrint( + "connect to server fail from insert sub thread, reason: %s\n", + taos_errstr(NULL)); + exit(-1); + } + } else { + t_info->taos = NULL; + } + + if ((NULL == superTblInfo) + || (0 == superTblInfo->multiThreadWriteOneTbl)) { + t_info->start_table_from = startFrom; + t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = t_info->end_table_to + 1; + } else { + t_info->start_table_from = 0; + t_info->ntables = superTblInfo->childTblCount; + t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint(); + } + + tsem_init(&(t_info->lock_sem), 0, 0); + if (SYNC == g_Dbs.queryMode) { + pthread_create(pids + i, NULL, syncWrite, t_info); + } else { + pthread_create(pids + i, NULL, asyncWrite, t_info); + } } - + for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); } - double t = getCurrentTime() - ts; - printf("Spent %.4f seconds to create %d tables with %d connections\n", t, ntables, threads); + int64_t totalDelay = 0; + int64_t maxDelay = 0; + int64_t minDelay = INT16_MAX; + int64_t cntDelay = 1; + double avgDelay = 0; for (int i = 0; i < threads; i++) { - info *t_info = infos + i; - tsem_destroy(&(t_info->mutex_sem)); + threadInfo *t_info = infos + i; + tsem_destroy(&(t_info->lock_sem)); + taos_close(t_info->taos); + + debugPrint("%s() LN%d, [%d] totalInsert=%"PRId64" totalAffected=%"PRId64"\n", + __func__, __LINE__, + t_info->threadID, t_info->totalInsertRows, + t_info->totalAffectedRows); + if (superTblInfo) { + superTblInfo->totalAffectedRows += t_info->totalAffectedRows; + superTblInfo->totalInsertRows += t_info->totalInsertRows; + } else { + g_args.totalAffectedRows += t_info->totalAffectedRows; + g_args.totalInsertRows += t_info->totalInsertRows; + } + + totalDelay += t_info->totalDelay; + cntDelay += t_info->cntDelay; + if (t_info->maxDelay > maxDelay) maxDelay = t_info->maxDelay; + if (t_info->minDelay < minDelay) minDelay = t_info->minDelay; } + cntDelay -= 1; - free(pids); - free(infos); + if (cntDelay == 0) cntDelay = 1; + avgDelay = (double)totalDelay / cntDelay; + + double end = getCurrentTime(); + double t = end - start; + + if (superTblInfo) { + printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", + t, superTblInfo->totalInsertRows, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + superTblInfo->totalInsertRows / t); + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s. %2.f records/second\n\n", + t, superTblInfo->totalInsertRows, + superTblInfo->totalAffectedRows, + threads, db_name, superTblInfo->sTblName, + superTblInfo->totalInsertRows/ t); + } else { + printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n", + t, g_args.totalInsertRows, + g_args.totalAffectedRows, + threads, db_name, + g_args.totalInsertRows / t); + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s %2.f records/second\n\n", + t, g_args.totalInsertRows, + g_args.totalAffectedRows, + threads, db_name, + g_args.totalInsertRows / t); + } - return ; + printf("insert delay, avg: %10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); + fprintf(g_fpOfInsertResult, "insert delay, avg:%10.6fms, max: %10.6fms, min: %10.6fms\n\n", + avgDelay/1000.0, (double)maxDelay/1000.0, (double)minDelay/1000.0); + + //taos_close(taos); + + free(pids); + free(infos); } -void *readTable(void *sarg) { - info *rinfo = (info *)sarg; +static void *readTable(void *sarg) { +#if 1 + threadInfo *rinfo = (threadInfo *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; - int64_t sTime = rinfo->start_time; + uint64_t sTime = rinfo->start_time; char *tb_prefix = rinfo->tb_prefix; FILE *fp = fopen(rinfo->fp, "a"); if (NULL == fp) { - printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); + errorPrint( "fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); return NULL; } - - int num_of_DPT = rinfo->nrecords_per_table; - int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; + + int num_of_DPT; +/* if (rinfo->superTblInfo) { + num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; + } else { + */ + num_of_DPT = g_args.num_of_DPT; +// } + + int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int totalData = num_of_DPT * num_of_tables; - bool do_aggreFunc = rinfo->do_aggreFunc; + bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { @@ -1129,17 +5363,19 @@ void *readTable(void *sarg) { double totalT = 0; int count = 0; for (int i = 0; i < num_of_tables; i++) { - sprintf(command, "select %s from %s%d where ts>= %" PRId64, aggreFunc[j], tb_prefix, i, sTime); + sprintf(command, "select %s from %s%d where ts>= %" PRId64, + aggreFunc[j], tb_prefix, i, sTime); double t = getCurrentTime(); TAOS_RES *pSql = taos_query(taos, command); int32_t code = taos_errno(pSql); if (code != 0) { - fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); + errorPrint( "Failed to query:%s\n", taos_errstr(pSql)); taos_free_result(pSql); taos_close(taos); - exit(EXIT_FAILURE); + fclose(fp); + return NULL; } while (taos_fetch_row(pSql) != NULL) { @@ -1158,13 +5394,14 @@ void *readTable(void *sarg) { printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT); } fprintf(fp, "\n"); - fclose(fp); +#endif return NULL; } -void *readMetric(void *sarg) { - info *rinfo = (info *)sarg; +static void *readMetric(void *sarg) { +#if 1 + threadInfo *rinfo = (threadInfo *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; FILE *fp = fopen(rinfo->fp, "a"); @@ -1172,11 +5409,11 @@ void *readMetric(void *sarg) { printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); return NULL; } - - int num_of_DPT = rinfo->nrecords_per_table; - int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; + + int num_of_DPT = rinfo->superTblInfo->insertRows; + int num_of_tables = rinfo->ntables; // rinfo->end_table_to - rinfo->start_table_from + 1; int totalData = num_of_DPT * num_of_tables; - bool do_aggreFunc = rinfo->do_aggreFunc; + bool do_aggreFunc = g_Dbs.do_aggreFunc; int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; if (!do_aggreFunc) { @@ -1193,9 +5430,9 @@ void *readMetric(void *sarg) { for (int i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "areaid = %d", i); + sprintf(tempS, "t1 = %d", i); } else { - sprintf(tempS, " or areaid = %d ", i); + sprintf(tempS, " or t1 = %d ", i); } strcat(condition, tempS); @@ -1210,10 +5447,11 @@ void *readMetric(void *sarg) { int32_t code = taos_errno(pSql); if (code != 0) { - fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); + errorPrint( "Failed to query:%s\n", taos_errstr(pSql)); taos_free_result(pSql); taos_close(taos); - exit(1); + fclose(fp); + return NULL; } int count = 0; while (taos_fetch_row(pSql) != NULL) { @@ -1221,340 +5459,1073 @@ void *readMetric(void *sarg) { } t = getCurrentTime() - t; - fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", num_of_tables * num_of_DPT / t, t * 1000); - printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t); + fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", + num_of_tables * num_of_DPT / t, t * 1000); + printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t); + + taos_free_result(pSql); + } + fprintf(fp, "\n"); + } + fclose(fp); +#endif + return NULL; +} + + +static int insertTestProcess() { + + setupForAnsiEscape(); + int ret = printfInsertMeta(); + resetAfterAnsiEscape(); + + if (ret == -1) + exit(EXIT_FAILURE); + + debugPrint("%d result file: %s\n", __LINE__, g_Dbs.resultFile); + g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); + if (NULL == g_fpOfInsertResult) { + errorPrint( "Failed to open %s for save result\n", g_Dbs.resultFile); + return -1; + } + + printfInsertMetaToFile(g_fpOfInsertResult); + + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } + + init_rand_data(); + + // create database and super tables + if(createDatabasesAndStables() != 0) { + fclose(g_fpOfInsertResult); + return -1; + } + + // pretreatement + prepareSampleData(); + + double start; + double end; + + // create child tables + start = getCurrentTime(); + createChildTables(); + end = getCurrentTime(); + + if (g_totalChildTables > 0) { + printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + end - start, g_totalChildTables, g_Dbs.threadCountByCreateTbl); + fprintf(g_fpOfInsertResult, + "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", + end - start, g_totalChildTables, g_Dbs.threadCountByCreateTbl); + } + + taosMsleep(1000); + // create sub threads for inserting data + //start = getCurrentTime(); + for (int i = 0; i < g_Dbs.dbCount; i++) { + if (g_Dbs.db[i].superTblCount > 0) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + if (0 == g_Dbs.db[i].superTbls[j].insertRows) { + continue; + } + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + superTblInfo); + } + } else { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + NULL); + } + } + //end = getCurrentTime(); + + //int64_t totalInsertRows = 0; + //int64_t totalAffectedRows = 0; + //for (int i = 0; i < g_Dbs.dbCount; i++) { + // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + // totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows; + // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; + //} + //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalInsertRows, totalAffectedRows, g_Dbs.threadCount); + postFreeResource(); + + return 0; +} + +static void *superQueryProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + + if (winfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + winfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + winfo->taos = taos; + } + } + + char sqlStr[MAX_DB_NAME_SIZE + 5]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(winfo->taos); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return NULL; + } + + int64_t st = 0; + int64_t et = 0; + + int queryTimes = g_queryInfo.specifiedQueryInfo.queryTimes; + + while(queryTimes --) { + if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < + (int64_t)g_queryInfo.specifiedQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); + } + + st = taosGetTimestampUs(); + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + int64_t t1 = taosGetTimestampUs(); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); + } + selectAndGetResult(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], tmpFile); + int64_t t2 = taosGetTimestampUs(); + printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %f s\n", + taosGetSelfPthreadId(), (t2 - t1)/1000000.0); + } else { + int64_t t1 = taosGetTimestampUs(); + int retCode = postProceSql(g_queryInfo.host, + g_queryInfo.port, g_queryInfo.specifiedQueryInfo.sql[i]); + int64_t t2 = taosGetTimestampUs(); + printf("=[restful] thread[%"PRId64"] complete one sql, Spent %f s\n", + taosGetSelfPthreadId(), (t2 - t1)/1000000.0); + + if (0 != retCode) { + printf("====restful return fail, threadID[%d]\n", winfo->threadID); + return NULL; + } + } + } + et = taosGetTimestampUs(); + printf("==thread[%"PRId64"] complete all sqls to specify tables once queries duration:%.6fs\n\n", + taosGetSelfPthreadId(), (double)(et - st)/1000.0); + } + return NULL; +} + +static void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { + char sourceString[32] = "xxxx"; + char subTblName[MAX_TB_NAME_SIZE*3]; + sprintf(subTblName, "%s.%s", + g_queryInfo.dbName, + g_queryInfo.superQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); + + //printf("inSql: %s\n", inSql); + + char* pos = strstr(inSql, sourceString); + if (0 == pos) { + return; + } + + tstrncpy(outSql, inSql, pos - inSql + 1); + //printf("1: %s\n", outSql); + strcat(outSql, subTblName); + //printf("2: %s\n", outSql); + strcat(outSql, pos+strlen(sourceString)); + //printf("3: %s\n", outSql); +} + +static void *subQueryProcess(void *sarg) { + char sqlstr[1024]; + threadInfo *winfo = (threadInfo *)sarg; + + if (winfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + winfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + winfo->taos = taos; + } + } + + int64_t st = 0; + int64_t et = (int64_t)g_queryInfo.superQueryInfo.rate*1000; + + int queryTimes = g_queryInfo.superQueryInfo.queryTimes; + + while(queryTimes --) { + if (g_queryInfo.superQueryInfo.rate + && (et - st) < (int64_t)g_queryInfo.superQueryInfo.rate*1000) { + taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms + //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); + } + + st = taosGetTimestampUs(); + for (int i = winfo->start_table_from; i <= winfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + memset(sqlstr,0,sizeof(sqlstr)); + replaceSubTblName(g_queryInfo.superQueryInfo.sql[j], sqlstr, i); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[j][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[j], + winfo->threadID); + } + selectAndGetResult(winfo->taos, sqlstr, tmpFile); + } + } + et = taosGetTimestampUs(); + printf("####thread[%"PRId64"] complete all sqls to allocate all sub-tables[%d - %d] once queries duration:%.4fs\n\n", + taosGetSelfPthreadId(), + winfo->start_table_from, + winfo->end_table_to, + (double)(et - st)/1000000.0); + } + + return NULL; +} + +static int queryTestProcess() { + + setupForAnsiEscape(); + printfQueryMeta(); + resetAfterAnsiEscape(); + + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + NULL, + g_queryInfo.port); + if (taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", + taos_errstr(NULL)); + exit(-1); + } + + if (0 != g_queryInfo.superQueryInfo.sqlCount) { + getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.superQueryInfo.sTblName, + &g_queryInfo.superQueryInfo.childTblName, + &g_queryInfo.superQueryInfo.childTblCount); + } + + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void)getchar(); + } + + printfQuerySystemInfo(taos); + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from specify table + if (g_queryInfo.specifiedQueryInfo.sqlCount > 0 + && g_queryInfo.specifiedQueryInfo.concurrent > 0) { + + pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + if (NULL == pids) { + taos_close(taos); + ERROR_EXIT("memory allocation failed\n"); + } + infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + if (NULL == infos) { + taos_close(taos); + free(pids); + ERROR_EXIT("memory allocation failed for create threads\n"); + } + + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + verbosePrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + if (0 != queryDbExec(taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(taos); + free(infos); + free(pids); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return -1; + } + } + + t_info->taos = NULL;// TODO: workaround to use separate taos connection; + + pthread_create(pids + i, NULL, superQueryProcess, t_info); + } + } else { + g_queryInfo.specifiedQueryInfo.concurrent = 0; + } + + taos_close(taos); + + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + //==== create sub threads for query from all sub table of the super table + if ((g_queryInfo.superQueryInfo.sqlCount > 0) + && (g_queryInfo.superQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); + if (NULL == pidsOfSub) { + free(infos); + free(pids); + + ERROR_EXIT("memory allocation failed for create threads\n"); + } + + infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); + if (NULL == infosOfSub) { + free(pidsOfSub); + free(infos); + free(pids); + ERROR_EXIT("memory allocation failed for create threads\n"); + } + + int ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; + + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } + + int b = 0; + if (threads != 0) { + b = ntables % threads; + } + + int startFrom = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infosOfSub + i; + t_info->threadID = i; + + t_info->start_table_from = startFrom; + t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = t_info->end_table_to + 1; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info); + } + + g_queryInfo.superQueryInfo.threadCnt = threads; + } else { + g_queryInfo.superQueryInfo.threadCnt = 0; + } + + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { + pthread_join(pids[i], NULL); + } + + tmfree((char*)pids); + tmfree((char*)infos); + + for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } + + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); + +// taos_close(taos);// TODO: workaround to use separate taos connection; + return 0; +} + +static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to subscribe result, code:%d, reason:%s\n", + __func__, __LINE__, code, taos_errstr(res)); + return; + } + + getResult(res, (char*)param); + taos_free_result(res); +} + +static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) { + TAOS_SUB* tsub = NULL; + + if (g_queryInfo.specifiedQueryInfo.subscribeMode) { + tsub = taos_subscribe(taos, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + topic, sql, subscribe_callback, (void*)resultFileName, + g_queryInfo.specifiedQueryInfo.subscribeInterval); + } else { + tsub = taos_subscribe(taos, + g_queryInfo.specifiedQueryInfo.subscribeRestart, + topic, sql, NULL, NULL, 0); + } + + if (tsub == NULL) { + printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql); + return NULL; + } + + return tsub; +} + +static void *subSubscribeProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + char subSqlstr[1024]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; + + if (winfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + g_queryInfo.dbName, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + winfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + winfo->taos = taos; + } + } + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(winfo->taos); + errorPrint( "use database %s failed!\n\n", + g_queryInfo.dbName); + return NULL; + } + + //int64_t st = 0; + //int64_t et = 0; + do { + //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); + //} + + //st = taosGetTimestampMs(); + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + sprintf(topic, "taosdemo-subscribe-%d", i); + memset(subSqlstr,0,sizeof(subSqlstr)); + replaceSubTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[i], winfo->threadID); + } + tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); + if (NULL == tsub[i]) { + taos_close(winfo->taos); + return NULL; + } + } + //et = taosGetTimestampMs(); + //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); + } while (0); + + // start loop to consume result + TAOS_RES* res = NULL; + while (1) { + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.superQueryInfo.subscribeMode) { + continue; + } + + res = taos_consume(tsub[i]); + if (res) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[i], + winfo->threadID); + } + getResult(res, tmpFile); + } + } + } + taos_free_result(res); + + for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { + taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); + } + + taos_close(winfo->taos); + return NULL; +} + +static void *superSubscribeProcess(void *sarg) { + threadInfo *winfo = (threadInfo *)sarg; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; + + if (winfo->taos == NULL) { + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + g_queryInfo.dbName, + g_queryInfo.port); + if (taos == NULL) { + errorPrint("[%d] Failed to connect to TDengine, reason:%s\n", + winfo->threadID, taos_errstr(NULL)); + return NULL; + } else { + winfo->taos = taos; + } + } + + char sqlStr[MAX_TB_NAME_SIZE*2]; + sprintf(sqlStr, "use %s", g_queryInfo.dbName); + debugPrint("%s() %d sqlStr: %s\n", __func__, __LINE__, sqlStr); + if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE, false)) { + taos_close(winfo->taos); + return NULL; + } + + //int64_t st = 0; + //int64_t et = 0; + do { + //if (g_queryInfo.specifiedQueryInfo.rate && (et - st) < g_queryInfo.specifiedQueryInfo.rate*1000) { + // taosMsleep(g_queryInfo.specifiedQueryInfo.rate*1000 - (et - st)); // ms + // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_from, winfo->end_table_to); + //} + + //st = taosGetTimestampMs(); + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + sprintf(topic, "taosdemo-subscribe-%d", i); + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); + } + tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); + if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) { + taos_close(winfo->taos); + return NULL; + } + } + //et = taosGetTimestampMs(); + //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); + } while (0); + + // start loop to consume result + TAOS_RES* res = NULL; + while (1) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + if (1 == g_queryInfo.specifiedQueryInfo.subscribeMode) { + continue; + } + + res = taos_consume(tsub[i]); + if (res) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); + } + getResult(res, tmpFile); + } + } + } + taos_free_result(res); + + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + taos_unsubscribe(tsub[i], g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + } + + taos_close(winfo->taos); + return NULL; +} + +static int subscribeTestProcess() { + setupForAnsiEscape(); + printfQueryMeta(); + resetAfterAnsiEscape(); + + if (!g_args.answer_yes) { + printf("Press enter key to continue\n\n"); + (void) getchar(); + } + + TAOS * taos = NULL; + taos = taos_connect(g_queryInfo.host, + g_queryInfo.user, + g_queryInfo.password, + g_queryInfo.dbName, + g_queryInfo.port); + if (taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", + taos_errstr(NULL)); + exit(-1); + } + + if (0 != g_queryInfo.superQueryInfo.sqlCount) { + getAllChildNameOfSuperTable(taos, + g_queryInfo.dbName, + g_queryInfo.superQueryInfo.sTblName, + &g_queryInfo.superQueryInfo.childTblName, + &g_queryInfo.superQueryInfo.childTblCount); + } + + taos_close(taos); // TODO: workaround to use separate taos connection; + + pthread_t *pids = NULL; + threadInfo *infos = NULL; + //==== create sub threads for query from super table + if ((g_queryInfo.specifiedQueryInfo.sqlCount <= 0) || + (g_queryInfo.specifiedQueryInfo.concurrent <= 0)) { + errorPrint("%s() LN%d, query sqlCount %d or concurrent %d is not correct.\n", + __func__, __LINE__, g_queryInfo.specifiedQueryInfo.sqlCount, + g_queryInfo.specifiedQueryInfo.concurrent); + exit(-1); + } + + pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); + exit(-1); + } + + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { + threadInfo *t_info = infos + i; + t_info->threadID = i; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pids + i, NULL, superSubscribeProcess, t_info); + } - taos_free_result(pSql); + //==== create sub threads for query from sub table + pthread_t *pidsOfSub = NULL; + threadInfo *infosOfSub = NULL; + if ((g_queryInfo.superQueryInfo.sqlCount > 0) + && (g_queryInfo.superQueryInfo.threadCnt > 0)) { + pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + sizeof(pthread_t)); + infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + sizeof(threadInfo)); + if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", + __func__, __LINE__); + // taos_close(taos); + exit(-1); } - fprintf(fp, "\n"); - } - fclose(fp); - return NULL; -} + int ntables = g_queryInfo.superQueryInfo.childTblCount; + int threads = g_queryInfo.superQueryInfo.threadCnt; -static int queryDbExec(TAOS *taos, char *command, int type) { - int i; - TAOS_RES *res = NULL; - int32_t code = -1; + int a = ntables / threads; + if (a < 1) { + threads = ntables; + a = 1; + } - for (i = 0; i < 5; i++) { - if (NULL != res) { - taos_free_result(res); - res = NULL; + int b = 0; + if (threads != 0) { + b = ntables % threads; } - - res = taos_query(taos, command); - code = taos_errno(res); - if (0 == code) { - break; - } - } - if (code != 0) { - fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); - taos_free_result(res); - //taos_close(taos); - return -1; + int startFrom = 0; + for (int i = 0; i < threads; i++) { + threadInfo *t_info = infosOfSub + i; + t_info->threadID = i; + + t_info->start_table_from = startFrom; + t_info->ntables = iend_table_to = i < b ? startFrom + a : startFrom + a - 1; + startFrom = t_info->end_table_to + 1; + t_info->taos = NULL; // TODO: workaround to use separate taos connection; + pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); + } + + g_queryInfo.superQueryInfo.threadCnt = threads; + + for (int i = 0; i < g_queryInfo.superQueryInfo.threadCnt; i++) { + pthread_join(pidsOfSub[i], NULL); + } } - if (1 == type) { - int affectedRows = taos_affected_rows(res); - taos_free_result(res); - return affectedRows; + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { + pthread_join(pids[i], NULL); } - - taos_free_result(res); + + tmfree((char*)pids); + tmfree((char*)infos); + + tmfree((char*)pidsOfSub); + tmfree((char*)infosOfSub); +// taos_close(taos); return 0; } -void queryDB(TAOS *taos, char *command) { - int i; - TAOS_RES *pSql = NULL; - int32_t code = -1; +static void initOfInsertMeta() { + memset(&g_Dbs, 0, sizeof(SDbs)); - for (i = 0; i < 5; i++) { - if (NULL != pSql) { - taos_free_result(pSql); - pSql = NULL; - } - - pSql = taos_query(taos, command); - code = taos_errno(pSql); - if (0 == code) { - break; - } + // set default values + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + g_Dbs.port = 6030; + tstrncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); + tstrncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); + g_Dbs.threadCount = 2; + + g_Dbs.use_metric = g_args.use_metric; +} + +static void initOfQueryMeta() { + memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); + + // set default values + tstrncpy(g_queryInfo.host, "127.0.0.1", MAX_HOSTNAME_SIZE); + g_queryInfo.port = 6030; + tstrncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_USERNAME_SIZE); + tstrncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_PASSWORD_SIZE); +} + +static void setParaFromArg(){ + if (g_args.host) { + tstrncpy(g_Dbs.host, g_args.host, MAX_HOSTNAME_SIZE); + } else { + tstrncpy(g_Dbs.host, "127.0.0.1", MAX_HOSTNAME_SIZE); } - if (code != 0) { - fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); - taos_free_result(pSql); - taos_close(taos); - exit(EXIT_FAILURE); + if (g_args.user) { + tstrncpy(g_Dbs.user, g_args.user, MAX_USERNAME_SIZE); } - taos_free_result(pSql); -} + if (g_args.password) { + tstrncpy(g_Dbs.password, g_args.password, MAX_PASSWORD_SIZE); + } -// sync insertion -void *syncWrite(void *sarg) { - info *winfo = (info *)sarg; - char buffer[BUFFER_SIZE] = "\0"; - char data[MAX_DATA_SIZE]; - char **data_type = winfo->datatype; - int len_of_binary = winfo->len_of_binary; - int ncols_per_record = winfo->ncols_per_record; - srand((uint32_t)time(NULL)); - int64_t time_counter = winfo->start_time; - for (int i = 0; i < winfo->nrecords_per_table;) { - for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { - int inserted = i; - int64_t tmp_time = time_counter; - - char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, tID); - int k; - for (k = 0; k < winfo->nrecords_per_request;) { - int rand_num = rand() % 100; - int len = -1; - if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) { - int64_t d = tmp_time - rand() % 1000000 + rand_num; - len = generateData(data, data_type, ncols_per_record, d, len_of_binary); - } else { - len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary); - } + if (g_args.port) { + g_Dbs.port = g_args.port; + } - //assert(len + pstr - buffer < BUFFER_SIZE); - if (len + pstr - buffer >= BUFFER_SIZE) { // too long - break; - } + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + + g_Dbs.dbCount = 1; + g_Dbs.db[0].drop = 1; - pstr += sprintf(pstr, " %s", data); - inserted++; - k++; + tstrncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); + g_Dbs.db[0].dbCfg.replica = g_args.replica; + tstrncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); - if (inserted >= winfo->nrecords_per_table) break; + tstrncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); + + g_Dbs.use_metric = g_args.use_metric; + g_Dbs.insert_only = g_args.insert_only; + + g_Dbs.do_aggreFunc = true; + + char dataString[STRING_LEN]; + char **data_type = g_args.datatype; + + memset(dataString, 0, STRING_LEN); + + if (strcasecmp(data_type[0], "BINARY") == 0 + || strcasecmp(data_type[0], "BOOL") == 0 + || strcasecmp(data_type[0], "NCHAR") == 0 ) { + g_Dbs.do_aggreFunc = false; + } + + if (g_args.use_metric) { + g_Dbs.db[0].superTblCount = 1; + tstrncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; + g_Dbs.threadCount = g_args.num_of_threads; + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.queryMode = g_args.mode; + + g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; + g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; + g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; + g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; + tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, + g_args.tb_prefix, MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); + tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, + "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; + + g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; + g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; + + g_Dbs.db[0].superTbls[0].columnCount = 0; + for (int i = 0; i < MAX_NUM_DATATYPE; i++) { + if (data_type[i] == NULL) { + break; } - /* puts(buffer); */ - int64_t startTs; - int64_t endTs; - startTs = taosGetTimestampUs(); - //queryDB(winfo->taos, buffer); - int affectedRows = queryDbExec(winfo->taos, buffer, 1); - - if (0 <= affectedRows){ - endTs = taosGetTimestampUs(); - int64_t delay = endTs - startTs; - if (delay > winfo->maxDelay) winfo->maxDelay = delay; - if (delay < winfo->minDelay) winfo->minDelay = delay; - winfo->cntDelay++; - winfo->totalDelay += delay; - //winfo->avgDelay = (double)winfo->totalDelay / winfo->cntDelay; - } + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, + data_type[i], MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].columnCount++; + } - if (tID == winfo->end_table_id) { - i = inserted; - time_counter = tmp_time; + if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { + g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; + } else { + for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { + tstrncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; + g_Dbs.db[0].superTbls[0].columnCount++; } } + + tstrncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; + + tstrncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); + g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; + g_Dbs.db[0].superTbls[0].tagCount = 2; + } else { + g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; + g_Dbs.db[0].superTbls[0].tagCount = 0; } - return NULL; } -void *asyncWrite(void *sarg) { - info *winfo = (info *)sarg; - taos_query_a(winfo->taos, "show databases", callBack, winfo); +/* Function to do regular expression check */ +static int regexMatch(const char *s, const char *reg, int cflags) { + regex_t regex; + char msgbuf[100] = {0}; - tsem_wait(&(winfo->lock_sem)); + /* Compile regular expression */ + if (regcomp(®ex, reg, cflags) != 0) { + printf("Fail to compile regex\n"); + exit(-1); + } - return NULL; + /* Execute regular expression */ + int reti = regexec(®ex, s, 0, NULL, 0); + if (!reti) { + regfree(®ex); + return 1; + } else if (reti == REG_NOMATCH) { + regfree(®ex); + return 0; + } else { + regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); + printf("Regex match failed: %s\n", msgbuf); + regfree(®ex); + exit(-1); + } + + return 0; } -void callBack(void *param, TAOS_RES *res, int code) { - info* winfo = (info*)param; - char **datatype = winfo->datatype; - int ncols_per_record = winfo->ncols_per_record; - int len_of_binary = winfo->len_of_binary; +static int isCommentLine(char *line) { + if (line == NULL) return 1; - int64_t tmp_time = winfo->start_time; - char *buffer = calloc(1, BUFFER_SIZE); - char *data = calloc(1, MAX_DATA_SIZE); - char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id); - if (winfo->counter >= winfo->nrecords_per_table) { - winfo->start_table_id++; - winfo->counter = 0; - } - if (winfo->start_table_id > winfo->end_table_id) { - tsem_post(&winfo->lock_sem); - free(buffer); - free(data); - taos_free_result(res); + return regexMatch(line, "^\\s*#.*", REG_EXTENDED); +} + +static void querySqlFile(TAOS* taos, char* sqlFile) +{ + FILE *fp = fopen(sqlFile, "r"); + if (fp == NULL) { + printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); return; } - - for (int i = 0; i < winfo->nrecords_per_request; i++) { - int rand_num = rand() % 100; - if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) - { - int64_t d = tmp_time - rand() % 1000000 + rand_num; - generateData(data, datatype, ncols_per_record, d, len_of_binary); - } else - { - generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary); + + int read_len = 0; + char * cmd = calloc(1, MAX_SQL_SIZE); + size_t cmd_len = 0; + char * line = NULL; + size_t line_len = 0; + + double t = getCurrentTime(); + + while ((read_len = tgetline(&line, &line_len, fp)) != -1) { + if (read_len >= MAX_SQL_SIZE) continue; + line[--read_len] = '\0'; + + if (read_len == 0 || isCommentLine(line)) { // line starts with # + continue; } - pstr += sprintf(pstr, "%s", data); - winfo->counter++; - if (winfo->counter >= winfo->nrecords_per_table) { - break; + if (line[read_len - 1] == '\\') { + line[read_len - 1] = ' '; + memcpy(cmd + cmd_len, line, read_len); + cmd_len += read_len; + continue; + } + + memcpy(cmd + cmd_len, line, read_len); + verbosePrint("%s() LN%d cmd: %s\n", __func__, __LINE__, cmd); + if (0 != queryDbExec(taos, cmd, NO_INSERT_TYPE, false)) { + errorPrint("%s() LN%d, queryDbExec %s failed!\n", + __func__, __LINE__, cmd); + tmfree(cmd); + tmfree(line); + tmfclose(fp); + return; } + memset(cmd, 0, MAX_SQL_SIZE); + cmd_len = 0; } - taos_query_a(winfo->taos, buffer, callBack, winfo); - free(buffer); - free(data); - taos_free_result(res); + t = getCurrentTime() - t; + printf("run %s took %.6f second(s)\n\n", sqlFile, t); + + tmfree(cmd); + tmfree(line); + tmfclose(fp); + return; } -double getCurrentTime() { - struct timeval tv; - if (gettimeofday(&tv, NULL) != 0) { - perror("Failed to get current time in ms"); - exit(EXIT_FAILURE); - } +static void testMetaFile() { + if (INSERT_TEST == g_args.test_mode) { + if (g_Dbs.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); - return tv.tv_sec + tv.tv_usec / 1E6; -} + insertTestProcess(); -int32_t randint[MAX_PREPARED_RAND]; -int64_t randbigint[MAX_PREPARED_RAND]; -float randfloat[MAX_PREPARED_RAND]; -double randdouble[MAX_PREPARED_RAND]; + } else if (QUERY_TEST == g_args.test_mode) { + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); -int32_t rand_tinyint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 128; + queryTestProcess(); -} + } else if (SUBSCRIBE_TEST == g_args.test_mode) { + if (g_queryInfo.cfgDir[0]) + taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); -int32_t rand_smallint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 32767; -} + subscribeTestProcess(); -int32_t rand_int(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor]; + } else { + ; + } } -int64_t rand_bigint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randbigint[cursor]; - -} +static void queryResult() { + // select + if (false == g_Dbs.insert_only) { + // query data + + pthread_t read_id; + threadInfo *rInfo = malloc(sizeof(threadInfo)); + rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 + rInfo->start_table_from = 0; + + //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; + if (g_args.use_metric) { + rInfo->ntables = g_Dbs.db[0].superTbls[0].childTblCount; + rInfo->end_table_to = g_Dbs.db[0].superTbls[0].childTblCount - 1; + rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; + tstrncpy(rInfo->tb_prefix, + g_Dbs.db[0].superTbls[0].childTblPrefix, MAX_TB_NAME_SIZE); + } else { + rInfo->ntables = g_args.num_of_tables; + rInfo->end_table_to = g_args.num_of_tables -1; + tstrncpy(rInfo->tb_prefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); + } -float rand_float(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randfloat[cursor]; - -} + rInfo->taos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + if (rInfo->taos == NULL) { + errorPrint( "Failed to connect to TDengine, reason:%s\n", + taos_errstr(NULL)); + free(rInfo); + exit(-1); + } -double rand_double() { - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randdouble[cursor]; + tstrncpy(rInfo->fp, g_Dbs.resultFile, MAX_FILE_NAME_LEN); + if (!g_Dbs.use_metric) { + pthread_create(&read_id, NULL, readTable, rInfo); + } else { + pthread_create(&read_id, NULL, readMetric, rInfo); + } + pthread_join(read_id, NULL); + taos_close(rInfo->taos); + free(rInfo); + } } -void init_rand_data(){ - for (int i = 0; i < MAX_PREPARED_RAND; i++){ - randint[i] = (int)(rand() % 10); - randbigint[i] = (int64_t)(rand() % 2147483648); - randfloat[i] = (float)(rand() / 1000.0); - randdouble[i] = (double)(rand() / 1000000.0); - } +static void testCmdLine() { + + g_args.test_mode = INSERT_TEST; + insertTestProcess(); + + if (g_Dbs.insert_only) + return; + else + queryResult(); } -int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) { - memset(res, 0, MAX_DATA_SIZE); - char *pstr = res; - pstr += sprintf(pstr, "(%" PRId64, timestamp); - int c = 0; +int main(int argc, char *argv[]) { + parse_args(argc, argv, &g_args); - for (; c < MAX_NUM_DATATYPE; c++) { - if (data_type[c] == NULL) { - break; - } - } + debugPrint("meta file: %s\n", g_args.metaFile); - if (0 == c) { - perror("data type error!"); - exit(-1); - } + if (g_args.metaFile) { + initOfInsertMeta(); + initOfQueryMeta(); - for (int i = 0; i < num_of_cols; i++) { - if (strcasecmp(data_type[i % c], "tinyint") == 0) { - pstr += sprintf(pstr, ", %d", rand_tinyint() ); - } else if (strcasecmp(data_type[i % c], "smallint") == 0) { - pstr += sprintf(pstr, ", %d", rand_smallint()); - } else if (strcasecmp(data_type[i % c], "int") == 0) { - pstr += sprintf(pstr, ", %d", rand_int()); - } else if (strcasecmp(data_type[i % c], "bigint") == 0) { - pstr += sprintf(pstr, ", %" PRId64, rand_bigint()); - } else if (strcasecmp(data_type[i % c], "float") == 0) { - pstr += sprintf(pstr, ", %10.4f", rand_float()); - } else if (strcasecmp(data_type[i % c], "double") == 0) { - double t = rand_double(); - pstr += sprintf(pstr, ", %20.8f", t); - } else if (strcasecmp(data_type[i % c], "bool") == 0) { - bool b = rand() & 1; - pstr += sprintf(pstr, ", %s", b ? "true" : "false"); - } else if (strcasecmp(data_type[i % c], "binary") == 0) { - char *s = malloc(len_of_binary); - rand_string(s, len_of_binary); - pstr += sprintf(pstr, ", \"%s\"", s); - free(s); - }else if (strcasecmp(data_type[i % c], "nchar") == 0) { - char *s = malloc(len_of_binary); - rand_string(s, len_of_binary); - pstr += sprintf(pstr, ", \"%s\"", s); - free(s); + if (false == getInfoFromJsonFile(g_args.metaFile)) { + printf("Failed to read %s\n", g_args.metaFile); + return 1; } - if (pstr - res > MAX_DATA_SIZE) { - perror("column length too long, abort"); - exit(-1); + testMetaFile(); + } else { + memset(&g_Dbs, 0, sizeof(SDbs)); + setParaFromArg(); + + if (NULL != g_args.sqlFile) { + TAOS* qtaos = taos_connect( + g_Dbs.host, + g_Dbs.user, + g_Dbs.password, + g_Dbs.db[0].dbName, + g_Dbs.port); + querySqlFile(qtaos, g_args.sqlFile); + taos_close(qtaos); + + } else { + testCmdLine(); } } - pstr += sprintf(pstr, ")"); - - return (int32_t)(pstr - res); + return 0; } -static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890"; -void rand_string(char *str, int size) { - str[0] = 0; - if (size > 0) { - --size; - int n; - for (n = 0; n < size; n++) { - int key = rand() % (int)(sizeof charset - 1); - str[n] = charset[key]; - } - str[n] = 0; - } -} diff --git a/src/kit/taosdemox/CMakeLists.txt b/src/kit/taosdemox/CMakeLists.txt deleted file mode 100644 index 7db6c04b281aa6dea3e9b51fc47a8796fd64371b..0000000000000000000000000000000000000000 --- a/src/kit/taosdemox/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) -PROJECT(TDengine) - -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/include) - -IF (TD_LINUX) - AUX_SOURCE_DIRECTORY(. SRC) - ADD_EXECUTABLE(taosdemox ${SRC}) - - #find_program(HAVE_CURL NAMES curl) - IF ((NOT TD_ARM_64) AND (NOT TD_ARM_32)) - ADD_DEFINITIONS(-DTD_LOWA_CURL) - LINK_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/libcurl/lib) - ADD_LIBRARY(curl STATIC IMPORTED) - SET_PROPERTY(TARGET curl PROPERTY IMPORTED_LOCATION ${TD_COMMUNITY_DIR}/deps/libcurl/lib/libcurl.a) - TARGET_LINK_LIBRARIES(taosdemox curl) - ENDIF () - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(taosdemox taos_static cJson) - ELSE () - TARGET_LINK_LIBRARIES(taosdemox taos cJson) - ENDIF () -ENDIF () diff --git a/src/kit/taosdemox/query.json b/src/kit/taosdemox/query.json deleted file mode 100644 index b7b08edfc912bdccc12bc6b6672d62a8ee4ad417..0000000000000000000000000000000000000000 --- a/src/kit/taosdemox/query.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "filetype":"query", - "cfgdir": "/etc/taos", - "host": "127.0.0.1", - "port": 6030, - "user": "root", - "password": "taosdata", - "databases": "db01", - "specified_table_query": - {"query_interval":1, "concurrent":1, - "sqls": [{"sql": "select count(*) from stb01", "result": "./query_res0.txt"}] - }, - "super_table_query": - {"stblname": "stb01", "query_interval":1, "threads":1, - "sqls": [{"sql": "select count(*) from xxxx", "result": "./query_res1.txt"}] - } -} diff --git a/src/kit/taosdemox/taosdemox.c b/src/kit/taosdemox/taosdemox.c deleted file mode 100644 index 5c187030c521193e024062763c342cd0d928133c..0000000000000000000000000000000000000000 --- a/src/kit/taosdemox/taosdemox.c +++ /dev/null @@ -1,4701 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - - -/* - when in some thread query return error, thread don't exit, but return, otherwise coredump in other thread. -*/ - -#define _GNU_SOURCE -#define CURL_STATICLIB - -#ifdef TD_LOWA_CURL -#include "curl/curl.h" -#endif - -#ifdef LINUX - #include "os.h" - #include "cJSON.h" - #include - #include - #include - #ifndef _ALPINE - #include - #endif - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include - #include -#else - #include - #include - #include - #include "os.h" - - #pragma comment ( lib, "libcurl.lib" ) - #pragma comment ( lib, "ws2_32.lib" ) - #pragma comment ( lib, "winmm.lib" ) - #pragma comment ( lib, "wldap32.lib" ) -#endif - -#include "taos.h" -#include "tutil.h" - -extern char configDir[]; - -#define INSERT_JSON_NAME "insert.json" -#define QUERY_JSON_NAME "query.json" -#define SUBSCRIBE_JSON_NAME "subscribe.json" - -#define INSERT_MODE 0 -#define QUERY_MODE 1 -#define SUBSCRIBE_MODE 2 - -#define MAX_SQL_SIZE 65536 -#define BUFFER_SIZE (65536*2) -#define MAX_DB_NAME_SIZE 64 -#define MAX_TB_NAME_SIZE 64 -#define MAX_DATA_SIZE 16000 -#define MAX_NUM_DATATYPE 10 -#define OPT_ABORT 1 /* –abort */ -#define STRING_LEN 60000 -#define MAX_PREPARED_RAND 1000000 -//#define MAX_SQL_SIZE 65536 -#define MAX_FILE_NAME_LEN 256 - -#define MAX_SAMPLES_ONCE_FROM_FILE 10000 -#define MAX_NUM_DATATYPE 10 - -#define MAX_DB_COUNT 8 -#define MAX_SUPER_TABLE_COUNT 8 -#define MAX_COLUMN_COUNT 1024 -#define MAX_TAG_COUNT 128 - -#define MAX_QUERY_SQL_COUNT 10 -#define MAX_QUERY_SQL_LENGTH 256 - -typedef enum CREATE_SUB_TALBE_MOD_EN { - PRE_CREATE_SUBTBL, - AUTO_CREATE_SUBTBL, - NO_CREATE_SUBTBL -} CREATE_SUB_TALBE_MOD_EN; - -typedef enum TALBE_EXISTS_EN { - TBL_ALREADY_EXISTS, - TBL_NO_EXISTS, - TBL_EXISTS_BUTT -} TALBE_EXISTS_EN; - -enum MODE { - SYNC, - ASYNC, - MODE_BUT -}; - -enum QUERY_TYPE { - NO_INSERT_TYPE, - INSERT_TYPE, - QUERY_TYPE_BUT -} ; - -enum _describe_table_index { - TSDB_DESCRIBE_METRIC_FIELD_INDEX, - TSDB_DESCRIBE_METRIC_TYPE_INDEX, - TSDB_DESCRIBE_METRIC_LENGTH_INDEX, - TSDB_DESCRIBE_METRIC_NOTE_INDEX, - TSDB_MAX_DESCRIBE_METRIC -}; - -typedef struct { - char field[TSDB_COL_NAME_LEN + 1]; - char type[16]; - int length; - char note[128]; -} SColDes; - -/* Used by main to communicate with parse_opt. */ -typedef struct SArguments_S { - char * metaFile; - char * host; - uint16_t port; - char * user; - char * password; - char * database; - int replica; - char * tb_prefix; - char * sqlFile; - bool use_metric; - bool insert_only; - char * output_file; - int mode; - char * datatype[MAX_NUM_DATATYPE + 1]; - int len_of_binary; - int num_of_CPR; - int num_of_threads; - int num_of_RPR; - int num_of_tables; - int num_of_DPT; - int abort; - int disorderRatio; - int disorderRange; - int method_of_delete; - char ** arg_list; -} SArguments; - -typedef struct SColumn_S { - char field[TSDB_COL_NAME_LEN + 1]; - char dataType[MAX_TB_NAME_SIZE]; - int dataLen; - char note[128]; -} StrColumn; - -typedef struct SSuperTable_S { - char sTblName[MAX_TB_NAME_SIZE]; - int childTblCount; - bool superTblExists; // 0: no, 1: yes - bool childTblExists; // 0: no, 1: yes - int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table - char childTblPrefix[MAX_TB_NAME_SIZE]; - char dataSource[MAX_TB_NAME_SIZE]; // rand_gen or sample - char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful - int insertRate; // 0: unlimit > 0 rows/s - - int multiThreadWriteOneTbl; // 0: no, 1: yes - int numberOfTblInOneSql; // 0/1: one table, > 1: number of tbl - int rowsPerTbl; // - int disorderRatio; // 0: no disorder, >0: x% - int disorderRange; // ms or us by database precision - int maxSqlLen; // - - int64_t insertRows; // 0: no limit - int timeStampStep; - char startTimestamp[MAX_TB_NAME_SIZE]; // - char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json - char sampleFile[MAX_FILE_NAME_LEN]; - char tagsFile[MAX_FILE_NAME_LEN]; - - int columnCount; - StrColumn columns[MAX_COLUMN_COUNT]; - int tagCount; - StrColumn tags[MAX_TAG_COUNT]; - - char* childTblName; - char* colsOfCreatChildTable; - int lenOfOneRow; - int lenOfTagOfOneRow; - - char* sampleDataBuf; - int sampleDataBufSize; - //int sampleRowCount; - //int sampleUsePos; - - int tagSource; // 0: rand, 1: tag sample - char* tagDataBuf; - int tagSampleCount; - int tagUsePos; - - // statistics - int64_t totalRowsInserted; - int64_t totalAffectedRows; -} SSuperTable; - -typedef struct SDbCfg_S { -// int maxtablesPerVnode; - int minRows; - int maxRows; - int comp; - int walLevel; - int fsync; - int replica; - int update; - int keep; - int days; - int cache; - int blocks; - int quorum; - char precision[MAX_TB_NAME_SIZE]; -} SDbCfg; - -typedef struct SDataBase_S { - char dbName[MAX_DB_NAME_SIZE]; - int drop; // 0: use exists, 1: if exists, drop then new create - SDbCfg dbCfg; - int superTblCount; - SSuperTable superTbls[MAX_SUPER_TABLE_COUNT]; -} SDataBase; - -typedef struct SDbs_S { - char cfgDir[MAX_FILE_NAME_LEN]; - char host[MAX_DB_NAME_SIZE]; - uint16_t port; - char user[MAX_DB_NAME_SIZE]; - char password[MAX_DB_NAME_SIZE]; - char resultFile[MAX_FILE_NAME_LEN]; - bool use_metric; - bool insert_only; - bool do_aggreFunc; - bool queryMode; - - int threadCount; - int threadCountByCreateTbl; - int dbCount; - SDataBase db[MAX_DB_COUNT]; - - // statistics - int64_t totalRowsInserted; - int64_t totalAffectedRows; -} SDbs; - -typedef struct SuperQueryInfo_S { - int rate; // 0: unlimit > 0 loop/s - int concurrent; - int sqlCount; - int subscribeMode; // 0: sync, 1: async - int subscribeInterval; // ms - int subscribeRestart; - int subscribeKeepProgress; - char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; -} SuperQueryInfo; - -typedef struct SubQueryInfo_S { - char sTblName[MAX_TB_NAME_SIZE]; - int rate; // 0: unlimit > 0 loop/s - int threadCnt; - int subscribeMode; // 0: sync, 1: async - int subscribeInterval; // ms - int subscribeRestart; - int subscribeKeepProgress; - int childTblCount; - char childTblPrefix[MAX_TB_NAME_SIZE]; - int sqlCount; - char sql[MAX_QUERY_SQL_COUNT][MAX_QUERY_SQL_LENGTH]; - char result[MAX_QUERY_SQL_COUNT][MAX_FILE_NAME_LEN]; - TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT]; - - char* childTblName; -} SubQueryInfo; - -typedef struct SQueryMetaInfo_S { - char cfgDir[MAX_FILE_NAME_LEN]; - char host[MAX_DB_NAME_SIZE]; - uint16_t port; - char user[MAX_DB_NAME_SIZE]; - char password[MAX_DB_NAME_SIZE]; - char dbName[MAX_DB_NAME_SIZE]; - char queryMode[MAX_TB_NAME_SIZE]; // taosc, restful - - SuperQueryInfo superQueryInfo; - SubQueryInfo subQueryInfo; -} SQueryMetaInfo; - -typedef struct SThreadInfo_S { - TAOS *taos; - #ifdef TD_LOWA_CURL - CURL *curl_handle; - #endif - int threadID; - char db_name[MAX_DB_NAME_SIZE]; - char fp[4096]; - char tb_prefix[MAX_TB_NAME_SIZE]; - int start_table_id; - int end_table_id; - int data_of_rate; - int64_t start_time; - char* cols; - bool use_metric; - SSuperTable* superTblInfo; - - // for async insert - tsem_t lock_sem; - int64_t counter; - int64_t st; - int64_t et; - int64_t lastTs; - int nrecords_per_request; - - // statistics - int64_t totalRowsInserted; - int64_t totalAffectedRows; -} threadInfo; - -typedef struct curlMemInfo_S { - char *buf; - size_t sizeleft; - } curlMemInfo; - - - -#ifdef LINUX - /* The options we understand. */ - static struct argp_option options[] = { - {0, 'f', "meta file", 0, "The meta data to the execution procedure, if use -f, all others options invalid. Default is NULL.", 0}, - #ifdef _TD_POWER_ - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/power/'.", 1}, - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'powerdb'.", 2}, - #else - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 1}, - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 2}, - #endif - {0, 'h', "host", 0, "The host to connect to TDengine. Default is localhost.", 2}, - {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 2}, - {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, - {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, - {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 4}, - {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 4}, - {0, 's', "sql file", 0, "The select sql file.", 6}, - {0, 'M', 0, 0, "Use metric flag.", 4}, - {0, 'o', "outputfile", 0, "Direct output to the named file. Default is './output.txt'.", 6}, - {0, 'q', "query_mode", 0, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.", 4}, - {0, 'b', "type_of_cols", 0, "The data_type of columns, default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP.", 4}, - {0, 'w', "length_of_chartype", 0, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16", 4}, - {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 10.", 4}, - {0, 'T', "num_of_threads", 0, "The number of threads. Default is 10.", 4}, - // {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 100.", 4}, - {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 4}, - {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 10000.", 4}, - {0, 'x', 0, 0, "Not insert only flag.", 4}, - {0, 'O', "disorderRatio", 0, "Insert mode--0: In order, > 0: disorder ratio. Default is in order.", 4}, - {0, 'R', "disorderRang", 0, "Out of order data's range, ms, default is 1000.", 4}, - //{0, 'D', "delete database", 0, "if elete database if exists. 0: no, 1: yes, default is 1", 5}, - {0}}; - -/* Parse a single option. */ -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - // Get the input argument from argp_parse, which we know is a pointer to our arguments structure. - SArguments *arguments = state->input; - wordexp_t full_path; - char **sptr; - switch (key) { - case 'f': - arguments->metaFile = arg; - break; - case 'h': - arguments->host = arg; - break; - case 'p': - arguments->port = atoi(arg); - break; - case 'u': - arguments->user = arg; - break; - case 'P': - arguments->password = arg; - break; - case 'o': - arguments->output_file = arg; - break; - case 's': - arguments->sqlFile = arg; - break; - case 'q': - arguments->mode = atoi(arg); - break; - case 'T': - arguments->num_of_threads = atoi(arg); - break; - //case 'r': - // arguments->num_of_RPR = atoi(arg); - // break; - case 't': - arguments->num_of_tables = atoi(arg); - break; - case 'n': - arguments->num_of_DPT = atoi(arg); - break; - case 'd': - arguments->database = arg; - break; - case 'l': - arguments->num_of_CPR = atoi(arg); - break; - case 'b': - sptr = arguments->datatype; - if (strstr(arg, ",") == NULL) { - if (strcasecmp(arg, "INT") != 0 && strcasecmp(arg, "FLOAT") != 0 && - strcasecmp(arg, "TINYINT") != 0 && strcasecmp(arg, "BOOL") != 0 && - strcasecmp(arg, "SMALLINT") != 0 && strcasecmp(arg, "TIMESTAMP") != 0 && - strcasecmp(arg, "BIGINT") != 0 && strcasecmp(arg, "DOUBLE") != 0 && - strcasecmp(arg, "BINARY") != 0 && strcasecmp(arg, "NCHAR") != 0) { - argp_error(state, "Invalid data_type!"); - } - sptr[0] = arg; - } else { - int index = 0; - char *dupstr = strdup(arg); - char *running = dupstr; - char *token = strsep(&running, ","); - while (token != NULL) { - if (strcasecmp(token, "INT") != 0 && strcasecmp(token, "FLOAT") != 0 && - strcasecmp(token, "TINYINT") != 0 && strcasecmp(token, "BOOL") != 0 && - strcasecmp(token, "SMALLINT") != 0 && strcasecmp(token, "TIMESTAMP") != 0 && - strcasecmp(token, "BIGINT") != 0 && strcasecmp(token, "DOUBLE") != 0 && - strcasecmp(token, "BINARY") != 0 && strcasecmp(token, "NCHAR") != 0) { - argp_error(state, "Invalid data_type!"); - } - sptr[index++] = token; - token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; - } - } - break; - case 'w': - arguments->len_of_binary = atoi(arg); - break; - case 'm': - arguments->tb_prefix = arg; - break; - case 'M': - arguments->use_metric = true; - break; - case 'x': - arguments->insert_only = false; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); - wordfree(&full_path); - break; - case 'O': - arguments->disorderRatio = atoi(arg); - if (arguments->disorderRatio < 0 || arguments->disorderRatio > 100) - { - argp_error(state, "Invalid disorder ratio, should 1 ~ 100!"); - } - break; - case 'R': - arguments->disorderRange = atoi(arg); - break; - case 'a': - arguments->replica = atoi(arg); - if (arguments->replica > 3 || arguments->replica < 1) - { - arguments->replica = 1; - } - break; - //case 'D': - // arguments->method_of_delete = atoi(arg); - // break; - case OPT_ABORT: - arguments->abort = 1; - break; - case ARGP_KEY_ARG: - /*arguments->arg_list = &state->argv[state->next-1]; - state->next = state->argc;*/ - argp_usage(state); - break; - - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - -static struct argp argp = {options, parse_opt, 0, 0}; - -void parse_args(int argc, char *argv[], SArguments *arguments) { - argp_parse(&argp, argc, argv, 0, 0, arguments); - if (arguments->abort) { - #ifndef _ALPINE - error(10, 0, "ABORTED"); - #else - abort(); - #endif - } -} - -#else - void printHelp() { - char indent[10] = " "; - printf("%s%s\n", indent, "-f"); - printf("%s%s%s\n", indent, indent, "The meta file to the execution procedure. Default is './meta.json'."); - printf("%s%s\n", indent, "-c"); - printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); - } - - void parse_args(int argc, char *argv[], SArguments *arguments) { - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "-f") == 0) { - arguments->metaFile = argv[++i]; - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); - } else if (strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(EXIT_FAILURE); - } else { - fprintf(stderr, "wrong options\n"); - printHelp(); - exit(EXIT_FAILURE); - } - } - } -#endif - -static bool getInfoFromJsonFile(char* file); -//static int generateOneRowDataForStb(SSuperTable* stbInfo); -//static int getDataIntoMemForStb(SSuperTable* stbInfo); -static void init_rand_data(); -static int createDatabases(); -static void createChildTables(); -static int queryDbExec(TAOS *taos, char *command, int type); - -/* ************ Global variables ************ */ - -int32_t randint[MAX_PREPARED_RAND]; -int64_t randbigint[MAX_PREPARED_RAND]; -float randfloat[MAX_PREPARED_RAND]; -double randdouble[MAX_PREPARED_RAND]; -char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; - -SArguments g_args = {NULL, - "127.0.0.1", // host - 6030, // port - "root", // user - #ifdef _TD_POWER_ - "powerdb", // password - #else - "taosdata", // password - #endif - "test", // database - 1, // replica - "t", // tb_prefix - NULL, // sqlFile - false, // use_metric - true, // insert_only - "./output.txt", // output_file - 0, // mode : sync or async - { - "TINYINT", // datatype - "SMALLINT", - "INT", - "BIGINT", - "FLOAT", - "DOUBLE", - "BINARY", - "NCHAR", - "BOOL", - "TIMESTAMP" - }, - 16, // len_of_binary - 10, // num_of_CPR - 10, // num_of_connections/thread - 100, // num_of_RPR - 10000, // num_of_tables - 10000, // num_of_DPT - 0, // abort - 0, // disorderRatio - 1000, // disorderRange - 1, // method_of_delete - NULL // arg_list -}; - - -static int g_jsonType = 0; -static SDbs g_Dbs; -static int g_totalChildTables = 0; -static SQueryMetaInfo g_queryInfo; -static FILE * g_fpOfInsertResult = NULL; - - -void tmfclose(FILE *fp) { - if (NULL != fp) { - fclose(fp); - } -} - -void tmfree(char *buf) { - if (NULL != buf) { - free(buf); - } -} - -static int queryDbExec(TAOS *taos, char *command, int type) { - int i; - TAOS_RES *res = NULL; - int32_t code = -1; - - for (i = 0; i < 5; i++) { - if (NULL != res) { - taos_free_result(res); - res = NULL; - } - - res = taos_query(taos, command); - code = taos_errno(res); - if (0 == code) { - break; - } - } - - if (code != 0) { - fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(res)); - taos_free_result(res); - //taos_close(taos); - return -1; - } - - if (INSERT_TYPE == type) { - int affectedRows = taos_affected_rows(res); - taos_free_result(res); - return affectedRows; - } - - taos_free_result(res); - return 0; -} - -static void getResult(TAOS_RES *res, char* resultFileName) { - TAOS_ROW row = NULL; - int num_rows = 0; - int num_fields = taos_field_count(res); - TAOS_FIELD *fields = taos_fetch_fields(res); - - FILE *fp = NULL; - if (resultFileName[0] != 0) { - fp = fopen(resultFileName, "at"); - if (fp == NULL) { - fprintf(stderr, "failed to open result file: %s, result will not save to file\n", resultFileName); - } - } - - char* databuf = (char*) calloc(1, 100*1024*1024); - if (databuf == NULL) { - fprintf(stderr, "failed to malloc, warning: save result to file slowly!\n"); - return ; - } - - int totalLen = 0; - char temp[16000]; - - // fetch the records row by row - while ((row = taos_fetch_row(res))) { - if (totalLen >= 100*1024*1024 - 32000) { - if (fp) fprintf(fp, "%s", databuf); - totalLen = 0; - memset(databuf, 0, 100*1024*1024); - } - num_rows++; - int len = taos_print_row(temp, row, fields, num_fields); - len += sprintf(temp + len, "\n"); - //printf("query result:%s\n", temp); - memcpy(databuf + totalLen, temp, len); - totalLen += len; - } - - if (fp) fprintf(fp, "%s", databuf); - tmfclose(fp); - free(databuf); -} - -static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { - TAOS_RES *res = taos_query(taos, command); - if (res == NULL || taos_errno(res) != 0) { - printf("failed to sql:%s, reason:%s\n", command, taos_errstr(res)); - taos_free_result(res); - return; - } - - getResult(res, resultFileName); - taos_free_result(res); -} - -double getCurrentTime() { - struct timeval tv; - if (gettimeofday(&tv, NULL) != 0) { - perror("Failed to get current time in ms"); - return 0.0; - } - - return tv.tv_sec + tv.tv_usec / 1E6; -} - -static int32_t rand_bool(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 2; -} - -static int32_t rand_tinyint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 128; -} - -static int32_t rand_smallint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor] % 32767; -} - -static int32_t rand_int(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randint[cursor]; -} - -static int64_t rand_bigint(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randbigint[cursor]; - -} - -static float rand_float(){ - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randfloat[cursor]; -} - -static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; -void rand_string(char *str, int size) { - str[0] = 0; - if (size > 0) { - //--size; - int n; - for (n = 0; n < size; n++) { - int key = rand_tinyint() % (int)(sizeof(charset) - 1); - str[n] = charset[key]; - } - str[n] = 0; - } -} - -static double rand_double() { - static int cursor; - cursor++; - cursor = cursor % MAX_PREPARED_RAND; - return randdouble[cursor]; - -} - -static void init_rand_data() { - for (int i = 0; i < MAX_PREPARED_RAND; i++){ - randint[i] = (int)(rand() % 65535); - randbigint[i] = (int64_t)(rand() % 2147483648); - randfloat[i] = (float)(rand() / 1000.0); - randdouble[i] = (double)(rand() / 1000000.0); - } -} - -static void printfInsertMeta() { - printf("\033[1m\033[40;32m================ insert.json parse result START ================\033[0m\n"); - printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); - printf("user: \033[33m%s\033[0m\n", g_Dbs.user); - printf("password: \033[33m%s\033[0m\n", g_Dbs.password); - printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); - printf("thread count: \033[33m%d\033[0m\n", g_Dbs.threadCount); - - printf("database count: \033[33m%d\033[0m\n", g_Dbs.dbCount); - for (int i = 0; i < g_Dbs.dbCount; i++) { - printf("database[\033[33m%d\033[0m]:\n", i); - printf(" database name: \033[33m%s\033[0m\n", g_Dbs.db[i].dbName); - if (0 == g_Dbs.db[i].drop) { - printf(" drop: \033[33mno\033[0m\n"); - }else { - printf(" drop: \033[33myes\033[0m\n"); - } - - if (g_Dbs.db[i].dbCfg.blocks > 0) { - printf(" blocks: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.blocks); - } - if (g_Dbs.db[i].dbCfg.cache > 0) { - printf(" cache: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.cache); - } - if (g_Dbs.db[i].dbCfg.days > 0) { - printf(" days: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.days); - } - if (g_Dbs.db[i].dbCfg.keep > 0) { - printf(" keep: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.keep); - } - if (g_Dbs.db[i].dbCfg.replica > 0) { - printf(" replica: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.replica); - } - if (g_Dbs.db[i].dbCfg.update > 0) { - printf(" update: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.update); - } - if (g_Dbs.db[i].dbCfg.minRows > 0) { - printf(" minRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.minRows); - } - if (g_Dbs.db[i].dbCfg.maxRows > 0) { - printf(" maxRows: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.maxRows); - } - if (g_Dbs.db[i].dbCfg.comp > 0) { - printf(" comp: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.comp); - } - if (g_Dbs.db[i].dbCfg.walLevel > 0) { - printf(" walLevel: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.walLevel); - } - if (g_Dbs.db[i].dbCfg.fsync > 0) { - printf(" fsync: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.fsync); - } - if (g_Dbs.db[i].dbCfg.quorum > 0) { - printf(" quorum: \033[33m%d\033[0m\n", g_Dbs.db[i].dbCfg.quorum); - } - if (g_Dbs.db[i].dbCfg.precision[0] != 0) { - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - printf(" precision: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); - } else { - printf(" precision error: \033[33m%s\033[0m\n", g_Dbs.db[i].dbCfg.precision); - exit(EXIT_FAILURE); - } - } - - printf(" super table count: \033[33m%d\033[0m\n", g_Dbs.db[i].superTblCount); - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - printf(" super table[\033[33m%d\033[0m]:\n", j); - - printf(" stbName: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sTblName); - - if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "no"); - } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "yes"); - } else { - printf(" autoCreateTable: \033[33m%s\033[0m\n", "error"); - } - - if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - printf(" childTblExists: \033[33m%s\033[0m\n", "no"); - } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - printf(" childTblExists: \033[33m%s\033[0m\n", "yes"); - } else { - printf(" childTblExists: \033[33m%s\033[0m\n", "error"); - } - - printf(" childTblCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); - printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); - printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); - printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); - printf(" insertRate: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].insertRate); - printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); - - if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { - printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); - }else { - printf(" multiThreadWriteOneTbl: \033[33myes\033[0m\n"); - } - printf(" numberOfTblInOneSql: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); - printf(" rowsPerTbl: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); - printf(" disorderRange: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRange); - printf(" disorderRatio: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].disorderRatio); - printf(" maxSqlLen: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].maxSqlLen); - - printf(" timeStampStep: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].timeStampStep); - printf(" startTimestamp: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].startTimestamp); - printf(" sampleFormat: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFormat); - printf(" sampleFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].sampleFile); - printf(" tagsFile: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].tagsFile); - - printf(" columnCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].columnCount); - for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { - //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { - printf("column[\033[33m%d\033[0m]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); - } else { - printf("column[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); - } - } - printf("\n"); - - printf(" tagCount: \033[33m%d\033[0m\n ", g_Dbs.db[i].superTbls[j].tagCount); - for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { - //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { - printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - } else { - printf("tag[%d]:\033[33m%s\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); - } - } - printf("\n"); - } - printf("\n"); - } - printf("\033[1m\033[40;32m================ insert.json parse result END================\033[0m\n"); -} - -static void printfInsertMetaToFile(FILE* fp) { - fprintf(fp, "================ insert.json parse result START================\n"); - fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); - fprintf(fp, "user: %s\n", g_Dbs.user); - fprintf(fp, "password: %s\n", g_Dbs.password); - fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); - fprintf(fp, "thread count: %d\n", g_Dbs.threadCount); - - fprintf(fp, "database count: %d\n", g_Dbs.dbCount); - for (int i = 0; i < g_Dbs.dbCount; i++) { - fprintf(fp, "database[%d]:\n", i); - fprintf(fp, " database name: %s\n", g_Dbs.db[i].dbName); - if (0 == g_Dbs.db[i].drop) { - fprintf(fp, " drop: no\n"); - }else { - fprintf(fp, " drop: yes\n"); - } - - if (g_Dbs.db[i].dbCfg.blocks > 0) { - fprintf(fp, " blocks: %d\n", g_Dbs.db[i].dbCfg.blocks); - } - if (g_Dbs.db[i].dbCfg.cache > 0) { - fprintf(fp, " cache: %d\n", g_Dbs.db[i].dbCfg.cache); - } - if (g_Dbs.db[i].dbCfg.days > 0) { - fprintf(fp, " days: %d\n", g_Dbs.db[i].dbCfg.days); - } - if (g_Dbs.db[i].dbCfg.keep > 0) { - fprintf(fp, " keep: %d\n", g_Dbs.db[i].dbCfg.keep); - } - if (g_Dbs.db[i].dbCfg.replica > 0) { - fprintf(fp, " replica: %d\n", g_Dbs.db[i].dbCfg.replica); - } - if (g_Dbs.db[i].dbCfg.update > 0) { - fprintf(fp, " update: %d\n", g_Dbs.db[i].dbCfg.update); - } - if (g_Dbs.db[i].dbCfg.minRows > 0) { - fprintf(fp, " minRows: %d\n", g_Dbs.db[i].dbCfg.minRows); - } - if (g_Dbs.db[i].dbCfg.maxRows > 0) { - fprintf(fp, " maxRows: %d\n", g_Dbs.db[i].dbCfg.maxRows); - } - if (g_Dbs.db[i].dbCfg.comp > 0) { - fprintf(fp, " comp: %d\n", g_Dbs.db[i].dbCfg.comp); - } - if (g_Dbs.db[i].dbCfg.walLevel > 0) { - fprintf(fp, " walLevel: %d\n", g_Dbs.db[i].dbCfg.walLevel); - } - if (g_Dbs.db[i].dbCfg.fsync > 0) { - fprintf(fp, " fsync: %d\n", g_Dbs.db[i].dbCfg.fsync); - } - if (g_Dbs.db[i].dbCfg.quorum > 0) { - fprintf(fp, " quorum: %d\n", g_Dbs.db[i].dbCfg.quorum); - } - if (g_Dbs.db[i].dbCfg.precision[0] != 0) { - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - fprintf(fp, " precision: %s\n", g_Dbs.db[i].dbCfg.precision); - } else { - fprintf(fp, " precision error: %s\n", g_Dbs.db[i].dbCfg.precision); - } - } - - fprintf(fp, " super table count: %d\n", g_Dbs.db[i].superTblCount); - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - fprintf(fp, " super table[%d]:\n", j); - - fprintf(fp, " stbName: %s\n", g_Dbs.db[i].superTbls[j].sTblName); - - if (PRE_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { - fprintf(fp, " autoCreateTable: %s\n", "no"); - } else if (AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) { - fprintf(fp, " autoCreateTable: %s\n", "yes"); - } else { - fprintf(fp, " autoCreateTable: %s\n", "error"); - } - - if (TBL_NO_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - fprintf(fp, " childTblExists: %s\n", "no"); - } else if (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists) { - fprintf(fp, " childTblExists: %s\n", "yes"); - } else { - fprintf(fp, " childTblExists: %s\n", "error"); - } - - fprintf(fp, " childTblCount: %d\n", g_Dbs.db[i].superTbls[j].childTblCount); - fprintf(fp, " childTblPrefix: %s\n", g_Dbs.db[i].superTbls[j].childTblPrefix); - fprintf(fp, " dataSource: %s\n", g_Dbs.db[i].superTbls[j].dataSource); - fprintf(fp, " insertMode: %s\n", g_Dbs.db[i].superTbls[j].insertMode); - fprintf(fp, " insertRate: %d\n", g_Dbs.db[i].superTbls[j].insertRate); - fprintf(fp, " insertRows: %"PRId64"\n", g_Dbs.db[i].superTbls[j].insertRows); - - if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { - fprintf(fp, " multiThreadWriteOneTbl: no\n"); - }else { - fprintf(fp, " multiThreadWriteOneTbl: yes\n"); - } - fprintf(fp, " numberOfTblInOneSql: %d\n", g_Dbs.db[i].superTbls[j].numberOfTblInOneSql); - fprintf(fp, " rowsPerTbl: %d\n", g_Dbs.db[i].superTbls[j].rowsPerTbl); - fprintf(fp, " disorderRange: %d\n", g_Dbs.db[i].superTbls[j].disorderRange); - fprintf(fp, " disorderRatio: %d\n", g_Dbs.db[i].superTbls[j].disorderRatio); - fprintf(fp, " maxSqlLen: %d\n", g_Dbs.db[i].superTbls[j].maxSqlLen); - - fprintf(fp, " timeStampStep: %d\n", g_Dbs.db[i].superTbls[j].timeStampStep); - fprintf(fp, " startTimestamp: %s\n", g_Dbs.db[i].superTbls[j].startTimestamp); - fprintf(fp, " sampleFormat: %s\n", g_Dbs.db[i].superTbls[j].sampleFormat); - fprintf(fp, " sampleFile: %s\n", g_Dbs.db[i].superTbls[j].sampleFile); - fprintf(fp, " tagsFile: %s\n", g_Dbs.db[i].superTbls[j].tagsFile); - - fprintf(fp, " columnCount: %d\n ", g_Dbs.db[i].superTbls[j].columnCount); - for (int k = 0; k < g_Dbs.db[i].superTbls[j].columnCount; k++) { - //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].columns[k].dataType, "nchar", 5))) { - fprintf(fp, "column[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType, g_Dbs.db[i].superTbls[j].columns[k].dataLen); - } else { - fprintf(fp, "column[%d]:%s ", k, g_Dbs.db[i].superTbls[j].columns[k].dataType); - } - } - fprintf(fp, "\n"); - - fprintf(fp, " tagCount: %d\n ", g_Dbs.db[i].superTbls[j].tagCount); - for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { - //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { - fprintf(fp, "tag[%d]:%s(%d) ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - } else { - fprintf(fp, "tag[%d]:%s ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType); - } - } - fprintf(fp, "\n"); - } - fprintf(fp, "\n"); - } - fprintf(fp, "================ insert.json parse result END ================\n\n"); -} - -static void printfQueryMeta() { - printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); - printf("host: \033[33m%s:%u\033[0m\n", g_queryInfo.host, g_queryInfo.port); - printf("user: \033[33m%s\033[0m\n", g_queryInfo.user); - printf("password: \033[33m%s\033[0m\n", g_queryInfo.password); - printf("database name: \033[33m%s\033[0m\n", g_queryInfo.dbName); - - printf("\n"); - printf("super table query info: \n"); - printf("rate: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.rate); - printf("concurrent: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.concurrent); - printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.sqlCount); - - if (SUBSCRIBE_MODE == g_jsonType) { - printf("mod: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeMode); - printf("interval: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); - printf("restart: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeRestart); - printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.superQueryInfo.subscribeKeepProgress); - } - - - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.superQueryInfo.sql[i]); - } - printf("\n"); - printf("sub table query info: \n"); - printf("rate: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.rate); - printf("threadCnt: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.threadCnt); - printf("childTblCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.childTblCount); - printf("childTblPrefix: \033[33m%s\033[0m\n", g_queryInfo.subQueryInfo.childTblPrefix); - - if (SUBSCRIBE_MODE == g_jsonType) { - printf("mod: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeMode); - printf("interval: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeInterval); - printf("restart: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeRestart); - printf("keepProgress: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.subscribeKeepProgress); - } - - printf("sqlCount: \033[33m%d\033[0m\n", g_queryInfo.subQueryInfo.sqlCount); - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - printf(" sql[%d]: \033[33m%s\033[0m\n", i, g_queryInfo.subQueryInfo.sql[i]); - } - printf("\n"); - printf("\033[1m\033[40;32m================ query.json parse result ================\033[0m\n"); -} - -#ifdef TD_LOWA_CURL -static size_t responseCallback(void *contents, size_t size, size_t nmemb, void *userp) -{ - size_t realsize = size * nmemb; - curlMemInfo* mem = (curlMemInfo*)userp; - - char *ptr = realloc(mem->buf, mem->sizeleft + realsize + 1); - if(ptr == NULL) { - /* out of memory! */ - printf("not enough memory (realloc returned NULL)\n"); - return 0; - } - - mem->buf = ptr; - memcpy(&(mem->buf[mem->sizeleft]), contents, realsize); - mem->sizeleft += realsize; - mem->buf[mem->sizeleft] = 0; - - //printf("result:%s\n\n", mem->buf); - - return realsize; -} - -void curlProceLogin(void) -{ - CURL *curl_handle; - CURLcode res; - - curlMemInfo chunk; - - chunk.buf = malloc(1); /* will be grown as needed by the realloc above */ - chunk.sizeleft = 0; /* no data at this point */ - - //curl_global_init(CURL_GLOBAL_ALL); - - /* init the curl session */ - curl_handle = curl_easy_init(); - - curl_easy_setopt(curl_handle,CURLOPT_POSTFIELDS,""); - curl_easy_setopt(curl_handle, CURLOPT_POST, 1); - - char dstUrl[128] = {0}; - snprintf(dstUrl, 128, "http://%s:6041/rest/login/root/taosdata", g_Dbs.host); - - /* specify URL to get */ - curl_easy_setopt(curl_handle, CURLOPT_URL, dstUrl); - - /* send all data to this function */ - curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, responseCallback); - - /* we pass our 'chunk' struct to the callback function */ - curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk); - - /* do it! */ - res = curl_easy_perform(curl_handle); - - /* check for errors */ - if(res != CURLE_OK) { - fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); - } - else { - //printf("response len:%lu, content: %s \n", (unsigned long)chunk.sizeleft, chunk.buf); - ; - } - - /* cleanup curl stuff */ - curl_easy_cleanup(curl_handle); - - free(chunk.buf); - - /* we're done with libcurl, so clean it up */ - //curl_global_cleanup(); - - return; -} - -int curlProceSql(char* host, uint16_t port, char* sqlstr, CURL *curl_handle) -{ - //curlProceLogin(); - - //CURL *curl_handle; - CURLcode res; - - curlMemInfo chunk; - - chunk.buf = malloc(1); /* will be grown as needed by the realloc above */ - chunk.sizeleft = 0; /* no data at this point */ - - - char dstUrl[128] = {0}; - snprintf(dstUrl, 128, "http://%s:%u/rest/sql", host, port+TSDB_PORT_HTTP); - - //curl_global_init(CURL_GLOBAL_ALL); - - /* init the curl session */ - //curl_handle = curl_easy_init(); - - //curl_easy_setopt(curl_handle,CURLOPT_POSTFIELDS,""); - curl_easy_setopt(curl_handle, CURLOPT_POST, 1L); - - /* specify URL to get */ - curl_easy_setopt(curl_handle, CURLOPT_URL, dstUrl); - - /* enable TCP keep-alive for this transfer */ - curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPALIVE, 1L); - /* keep-alive idle time to 120 seconds */ - curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPIDLE, 120L); - /* interval time between keep-alive probes: 60 seconds */ - curl_easy_setopt(curl_handle, CURLOPT_TCP_KEEPINTVL, 60L); - - /* send all data to this function */ - curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, responseCallback); - - /* we pass our 'chunk' struct to the callback function */ - curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk); - - struct curl_slist *list = NULL; - list = curl_slist_append(list, "Authorization: Basic cm9vdDp0YW9zZGF0YQ=="); - curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); - curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); - - /* Set the expected upload size. */ - curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)strlen(sqlstr)); - curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, sqlstr); - - /* get it! */ - res = curl_easy_perform(curl_handle); - - /* check for errors */ - if(res != CURLE_OK) { - fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); - return -1; - } - else { - /* curl_easy_perform() block end and return result */ - //printf("[%32.32s] sql response len:%lu, content: %s \n\n", sqlstr, (unsigned long)chunk.sizeleft, chunk.buf); - ; - } - - curl_slist_free_all(list); /* free the list again */ - - /* cleanup curl stuff */ - //curl_easy_cleanup(curl_handle); - - free(chunk.buf); - - /* we're done with libcurl, so clean it up */ - //curl_global_cleanup(); - - return 0; -} -#endif - -char* getTagValueFromTagSample( SSuperTable* stbInfo, int tagUsePos) { - char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); - if (NULL == dataBuf) { - printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); - return NULL; - } - - int dataLen = 0; - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "(%s)", stbInfo->tagDataBuf + stbInfo->lenOfTagOfOneRow * tagUsePos); - - return dataBuf; -} - -char* generateTagVaulesForStb(SSuperTable* stbInfo) { - char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); - if (NULL == dataBuf) { - printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); - return NULL; - } - - int dataLen = 0; - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "("); - for (int i = 0; i < stbInfo->tagCount; i++) { - if ((0 == strncasecmp(stbInfo->tags[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->tags[i].dataType, "nchar", 5))) { - if (stbInfo->tags[i].dataLen > TSDB_MAX_BINARY_LEN) { - printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); - tmfree(dataBuf); - return NULL; - } - - char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1); - if (NULL == buf) { - printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); - tmfree(dataBuf); - return NULL; - } - rand_string(buf, stbInfo->tags[i].dataLen); - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf); - tmfree(buf); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", 3)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_int()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", 6)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "float", 5)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_float()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "double", 6)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%f, ", rand_double()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "smallint", 8)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_smallint()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "tinyint", 7)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_tinyint()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bool", 4)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%d, ", rand_bool()); - } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "timestamp", 4)) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "%"PRId64", ", rand_bigint()); - } else { - printf("No support data type: %s\n", stbInfo->tags[i].dataType); - tmfree(dataBuf); - return NULL; - } - } - dataLen -= 2; - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, ")"); - return dataBuf; -} - -static int calcRowLen(SSuperTable* superTbls) { - int colIndex; - int lenOfOneRow = 0; - - for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { - char* dataType = superTbls->columns[colIndex].dataType; - - if (strcasecmp(dataType, "BINARY") == 0) { - lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; - } else if (strcasecmp(dataType, "NCHAR") == 0) { - lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; - } else if (strcasecmp(dataType, "INT") == 0) { - lenOfOneRow += 11; - } else if (strcasecmp(dataType, "BIGINT") == 0) { - lenOfOneRow += 21; - } else if (strcasecmp(dataType, "SMALLINT") == 0) { - lenOfOneRow += 6; - } else if (strcasecmp(dataType, "TINYINT") == 0) { - lenOfOneRow += 4; - } else if (strcasecmp(dataType, "BOOL") == 0) { - lenOfOneRow += 6; - } else if (strcasecmp(dataType, "FLOAT") == 0) { - lenOfOneRow += 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { - lenOfOneRow += 42; - } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { - lenOfOneRow += 21; - } else { - printf("get error data type : %s\n", dataType); - exit(-1); - } - } - - superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp - - int tagIndex; - int lenOfTagOfOneRow = 0; - for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { - char* dataType = superTbls->tags[tagIndex].dataType; - - if (strcasecmp(dataType, "BINARY") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "NCHAR") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "INT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; - } else if (strcasecmp(dataType, "BIGINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; - } else if (strcasecmp(dataType, "SMALLINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; - } else if (strcasecmp(dataType, "TINYINT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; - } else if (strcasecmp(dataType, "BOOL") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; - } else if (strcasecmp(dataType, "FLOAT") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; - } else { - printf("get error tag type : %s\n", dataType); - exit(-1); - } - } - - superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; - - return 0; -} - - -static int getAllChildNameOfSuperTable(TAOS * taos, char* dbName, char* sTblName, char** childTblNameOfSuperTbl, int* childTblCountOfSuperTbl) { - char command[BUFFER_SIZE] = "\0"; - TAOS_RES * res; - TAOS_ROW row = NULL; - - char* childTblName = *childTblNameOfSuperTbl; - - //get all child table name use cmd: select tbname from superTblName; - snprintf(command, BUFFER_SIZE, "select tbname from %s.%s", dbName, sTblName); - res = taos_query(taos, command); - int32_t code = taos_errno(res); - if (code != 0) { - printf("failed to run command %s\n", command); - taos_free_result(res); - taos_close(taos); - exit(-1); - } - - int childTblCount = 10000; - int count = 0; - childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); - char* pTblName = childTblName; - while ((row = taos_fetch_row(res)) != NULL) { - int32_t* len = taos_fetch_lengths(res); - strncpy(pTblName, (char *)row[0], len[0]); - //printf("==== sub table name: %s\n", pTblName); - count++; - if (count >= childTblCount - 1) { - char *tmp = realloc(childTblName, (size_t)childTblCount*1.5*TSDB_TABLE_NAME_LEN+1); - if (tmp != NULL) { - childTblName = tmp; - childTblCount = (int)(childTblCount*1.5); - memset(childTblName + count*TSDB_TABLE_NAME_LEN, 0, (size_t)((childTblCount-count)*TSDB_TABLE_NAME_LEN)); - } else { - // exit, if allocate more memory failed - printf("realloc fail for save child table name of %s.%s\n", dbName, sTblName); - tmfree(childTblName); - taos_free_result(res); - taos_close(taos); - exit(-1); - } - } - pTblName = childTblName + count * TSDB_TABLE_NAME_LEN; - } - - *childTblCountOfSuperTbl = count; - *childTblNameOfSuperTbl = childTblName; - - taos_free_result(res); - return 0; -} - -static int getSuperTableFromServer(TAOS * taos, char* dbName, SSuperTable* superTbls) { - char command[BUFFER_SIZE] = "\0"; - TAOS_RES * res; - TAOS_ROW row = NULL; - int count = 0; - - //get schema use cmd: describe superTblName; - snprintf(command, BUFFER_SIZE, "describe %s.%s", dbName, superTbls->sTblName); - res = taos_query(taos, command); - int32_t code = taos_errno(res); - if (code != 0) { - printf("failed to run command %s\n", command); - taos_free_result(res); - return -1; - } - - int tagIndex = 0; - int columnIndex = 0; - TAOS_FIELD *fields = taos_fetch_fields(res); - while ((row = taos_fetch_row(res)) != NULL) { - if (0 == count) { - count++; - continue; - } - - if (strcmp((char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], "TAG") == 0) { - strncpy(superTbls->tags[tagIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - strncpy(superTbls->tags[tagIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); - superTbls->tags[tagIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - strncpy(superTbls->tags[tagIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); - tagIndex++; - } else { - strncpy(superTbls->columns[columnIndex].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); - strncpy(superTbls->columns[columnIndex].dataType, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes); - superTbls->columns[columnIndex].dataLen = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); - strncpy(superTbls->columns[columnIndex].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes); - columnIndex++; - } - count++; - } - - superTbls->columnCount = columnIndex; - superTbls->tagCount = tagIndex; - taos_free_result(res); - - calcRowLen(superTbls); - - if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { - //get all child table name use cmd: select tbname from superTblName; - getAllChildNameOfSuperTable(taos, dbName, superTbls->sTblName, &superTbls->childTblName, &superTbls->childTblCount); - } - return 0; -} - -static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, bool use_metric) { - char command[BUFFER_SIZE] = "\0"; - - char cols[STRING_LEN] = "\0"; - int colIndex; - int len = 0; - - int lenOfOneRow = 0; - for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { - char* dataType = superTbls->columns[colIndex].dataType; - - if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "BINARY", superTbls->columns[colIndex].dataLen); - lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; - } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "NCHAR", superTbls->columns[colIndex].dataLen); - lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; - } else if (strcasecmp(dataType, "INT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); - lenOfOneRow += 11; - } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BIGINT"); - lenOfOneRow += 21; - } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "SMALLINT"); - lenOfOneRow += 6; - } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TINYINT"); - lenOfOneRow += 4; - } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "BOOL"); - lenOfOneRow += 6; - } else if (strcasecmp(dataType, "FLOAT") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "FLOAT"); - lenOfOneRow += 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "DOUBLE"); - lenOfOneRow += 42; - } else if (strcasecmp(dataType, "TIMESTAMP") == 0) { - len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "TIMESTAMP"); - lenOfOneRow += 21; - } else { - taos_close(taos); - printf("config error data type : %s\n", dataType); - exit(-1); - } - } - - superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp - //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow); - - // save for creating child table - superTbls->colsOfCreatChildTable = (char*)calloc(len+20, 1); - if (NULL == superTbls->colsOfCreatChildTable) { - printf("Failed when calloc, size:%d", len+1); - taos_close(taos); - exit(-1); - } - snprintf(superTbls->colsOfCreatChildTable, len+20, "(ts timestamp%s)", cols); - - if (use_metric) { - char tags[STRING_LEN] = "\0"; - int tagIndex; - len = 0; - - int lenOfTagOfOneRow = 0; - len += snprintf(tags + len, STRING_LEN - len, "("); - for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { - char* dataType = superTbls->tags[tagIndex].dataType; - - if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "BINARY", superTbls->tags[tagIndex].dataLen); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "NCHAR", superTbls->tags[tagIndex].dataLen); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "INT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "INT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; - } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BIGINT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; - } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "SMALLINT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; - } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "TINYINT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; - } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "BOOL"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; - } else if (strcasecmp(dataType, "FLOAT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "FLOAT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "DOUBLE"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; - } else { - taos_close(taos); - printf("config error tag type : %s\n", dataType); - exit(-1); - } - } - len -= 2; - len += snprintf(tags + len, STRING_LEN - len, ")"); - - superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; - - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s (ts timestamp%s) tags %s", dbName, superTbls->sTblName, cols, tags); - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { - return -1; - } - printf("\ncreate supertable %s success!\n\n", superTbls->sTblName); - } - return 0; -} - - -static int createDatabases() { - TAOS * taos = NULL; - int ret = 0; - taos_init(); - taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); - if (taos == NULL) { - fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - exit(-1); - } - char command[BUFFER_SIZE] = "\0"; - - - for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.db[i].drop) { - sprintf(command, "drop database if exists %s;", g_Dbs.db[i].dbName); - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { - taos_close(taos); - return -1; - } - } - - int dataLen = 0; - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "create database if not exists %s ", g_Dbs.db[i].dbName); - - if (g_Dbs.db[i].dbCfg.blocks > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "blocks %d ", g_Dbs.db[i].dbCfg.blocks); - } - if (g_Dbs.db[i].dbCfg.cache > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "cache %d ", g_Dbs.db[i].dbCfg.cache); - } - if (g_Dbs.db[i].dbCfg.days > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "days %d ", g_Dbs.db[i].dbCfg.days); - } - if (g_Dbs.db[i].dbCfg.keep > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "keep %d ", g_Dbs.db[i].dbCfg.keep); - } - if (g_Dbs.db[i].dbCfg.replica > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "replica %d ", g_Dbs.db[i].dbCfg.replica); - } - if (g_Dbs.db[i].dbCfg.update > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "update %d ", g_Dbs.db[i].dbCfg.update); - } - //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { - // dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); - //} - if (g_Dbs.db[i].dbCfg.minRows > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "minrows %d ", g_Dbs.db[i].dbCfg.minRows); - } - if (g_Dbs.db[i].dbCfg.maxRows > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "maxrows %d ", g_Dbs.db[i].dbCfg.maxRows); - } - if (g_Dbs.db[i].dbCfg.comp > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "comp %d ", g_Dbs.db[i].dbCfg.comp); - } - if (g_Dbs.db[i].dbCfg.walLevel > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "wal %d ", g_Dbs.db[i].dbCfg.walLevel); - } - if (g_Dbs.db[i].dbCfg.fsync > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "fsync %d ", g_Dbs.db[i].dbCfg.fsync); - } - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", 2)) || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "us", 2))) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, "precision \'%s\';", g_Dbs.db[i].dbCfg.precision); - } - - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { - taos_close(taos); - return -1; - } - printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); - - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - // describe super table, if exists - sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE)) { - g_Dbs.db[i].superTbls[j].superTblExists = TBL_NO_EXISTS; - ret = createSuperTable(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); - } else { - g_Dbs.db[i].superTbls[j].superTblExists = TBL_ALREADY_EXISTS; - ret = getSuperTableFromServer(taos, g_Dbs.db[i].dbName, &g_Dbs.db[i].superTbls[j]); - } - - if (0 != ret) { - taos_close(taos); - return -1; - } - } - } - - taos_close(taos); - return 0; -} - - -void * createTable(void *sarg) -{ - char command[BUFFER_SIZE] = "\0"; - - threadInfo *winfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = winfo->superTblInfo; - - int64_t lastPrintTime = taosGetTimestampMs(); - - //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id); - for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - if (0 == g_Dbs.use_metric) { - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable); - } else { - char* tagsValBuf = NULL; - if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo); - } else { - tagsValBuf = getTagValueFromTagSample(superTblInfo, i % superTblInfo->tagSampleCount); - } - if (NULL == tagsValBuf) { - return NULL; - } - snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.%s tags %s;", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf); - free(tagsValBuf); - } - - if (0 != queryDbExec(winfo->taos, command, NO_INSERT_TYPE)){ - return NULL; - } - - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] already create %d - %d tables\n", winfo->threadID, winfo->start_table_id, i); - lastPrintTime = currentPrintTime; - } - } - - return NULL; -} - -void startMultiThreadCreateChildTable(char* cols, int threads, int ntables, char* db_name, SSuperTable* superTblInfo) { - pthread_t *pids = malloc(threads * sizeof(pthread_t)); - threadInfo *infos = malloc(threads * sizeof(threadInfo)); - - if ((NULL == pids) || (NULL == infos)) { - printf("malloc failed\n"); - exit(-1); - } - - if (threads < 1) { - threads = 1; - } - - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int b = 0; - b = ntables % threads; - - int last = 0; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - t_info->superTblInfo = superTblInfo; - t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - t_info->start_table_id = last; - t_info->end_table_id = i < b ? last + a : last + a - 1; - last = t_info->end_table_id + 1; - t_info->use_metric = 1; - t_info->cols = cols; - pthread_create(pids + i, NULL, createTable, t_info); - } - - for (int i = 0; i < threads; i++) { - pthread_join(pids[i], NULL); - } - - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - taos_close(t_info->taos); - } - - free(pids); - free(infos); -} - - -static void createChildTables() { - for (int i = 0; i < g_Dbs.dbCount; i++) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { - continue; - } - startMultiThreadCreateChildTable(g_Dbs.db[i].superTbls[j].colsOfCreatChildTable, g_Dbs.threadCountByCreateTbl, g_Dbs.db[i].superTbls[j].childTblCount, g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); - g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - } - } -} - -/* -static int taosGetLineNum(const char *fileName) -{ - int lineNum = 0; - char cmd[1024] = { 0 }; - char buf[1024] = { 0 }; - sprintf(cmd, "wc -l %s", fileName); - - FILE *fp = popen(cmd, "r"); - if (fp == NULL) { - fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); - return lineNum; - } - - if (fgets(buf, sizeof(buf), fp)) { - int index = strchr((const char*)buf, ' ') - buf; - buf[index] = '\0'; - lineNum = atoi(buf); - } - pclose(fp); - return lineNum; -} -*/ - -/* - Read 10000 lines at most. If more than 10000 lines, continue to read after using -*/ -int readTagFromCsvFileToMem(SSuperTable * superTblInfo) { - size_t n = 0; - ssize_t readLen = 0; - char * line = NULL; - - FILE *fp = fopen(superTblInfo->tagsFile, "r"); - if (fp == NULL) { - printf("Failed to open tags file: %s, reason:%s\n", superTblInfo->tagsFile, strerror(errno)); - return -1; - } - - if (superTblInfo->tagDataBuf) { - free(superTblInfo->tagDataBuf); - superTblInfo->tagDataBuf = NULL; - } - - int tagCount = 10000; - int count = 0; - char* tagDataBuf = calloc(1, superTblInfo->lenOfTagOfOneRow * tagCount); - if (tagDataBuf == NULL) { - printf("Failed to calloc, reason:%s\n", strerror(errno)); - fclose(fp); - return -1; - } - - while ((readLen = getline(&line, &n, fp)) != -1) { - if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { - line[--readLen] = 0; - } - - if (readLen == 0) { - continue; - } - - memcpy(tagDataBuf + count * superTblInfo->lenOfTagOfOneRow, line, readLen); - count++; - - if (count >= tagCount - 1) { - char *tmp = realloc(tagDataBuf, (size_t)tagCount*1.5*superTblInfo->lenOfTagOfOneRow); - if (tmp != NULL) { - tagDataBuf = tmp; - tagCount = (int)(tagCount*1.5); - memset(tagDataBuf + count*superTblInfo->lenOfTagOfOneRow, 0, (size_t)((tagCount-count)*superTblInfo->lenOfTagOfOneRow)); - } else { - // exit, if allocate more memory failed - printf("realloc fail for save tag val from %s\n", superTblInfo->tagsFile); - tmfree(tagDataBuf); - free(line); - fclose(fp); - return -1; - } - } - } - - superTblInfo->tagDataBuf = tagDataBuf; - superTblInfo->tagSampleCount = count; - - free(line); - fclose(fp); - return 0; -} - -int readSampleFromJsonFileToMem(SSuperTable * superTblInfo) { - // TODO - return 0; -} - - -/* - Read 10000 lines at most. If more than 10000 lines, continue to read after using -*/ -int readSampleFromCsvFileToMem(FILE *fp, SSuperTable* superTblInfo, char* sampleBuf) { - size_t n = 0; - ssize_t readLen = 0; - char * line = NULL; - int getRows = 0; - - memset(sampleBuf, 0, MAX_SAMPLES_ONCE_FROM_FILE* superTblInfo->lenOfOneRow); - while (1) { - readLen = getline(&line, &n, fp); - if (-1 == readLen) { - if(0 != fseek(fp, 0, SEEK_SET)) { - printf("Failed to fseek file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); - return -1; - } - continue; - } - - if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { - line[--readLen] = 0; - } - - if (readLen == 0) { - continue; - } - - if (readLen > superTblInfo->lenOfOneRow) { - printf("sample row len[%d] overflow define schema len[%d], so discard this row\n", (int32_t)readLen, superTblInfo->lenOfOneRow); - continue; - } - - memcpy(sampleBuf + getRows * superTblInfo->lenOfOneRow, line, readLen); - getRows++; - - if (getRows == MAX_SAMPLES_ONCE_FROM_FILE) { - break; - } - } - - tmfree(line); - return 0; -} - -/* -void readSampleFromFileToMem(SSuperTable * supterTblInfo) { - int ret; - if (0 == strncasecmp(supterTblInfo->sampleFormat, "csv", 3)) { - ret = readSampleFromCsvFileToMem(supterTblInfo); - } else if (0 == strncasecmp(supterTblInfo->sampleFormat, "json", 4)) { - ret = readSampleFromJsonFileToMem(supterTblInfo); - } - - if (0 != ret) { - exit(-1); - } -} -*/ -static bool getColumnAndTagTypeFromInsertJsonFile(cJSON* stbInfo, SSuperTable* superTbls) { - bool ret = false; - - // columns - cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); - if (columns && columns->type != cJSON_Array) { - printf("failed to read json, columns not found\n"); - goto PARSE_OVER; - } else if (NULL == columns) { - superTbls->columnCount = 0; - superTbls->tagCount = 0; - return true; - } - - int columnSize = cJSON_GetArraySize(columns); - if (columnSize > MAX_COLUMN_COUNT) { - printf("failed to read json, column size overflow, max column size is %d\n", MAX_COLUMN_COUNT); - goto PARSE_OVER; - } - - int count = 1; - int index = 0; - StrColumn columnCase; - - //superTbls->columnCount = columnSize; - for (int k = 0; k < columnSize; ++k) { - cJSON* column = cJSON_GetArrayItem(columns, k); - if (column == NULL) continue; - - count = 1; - cJSON* countObj = cJSON_GetObjectItem(column, "count"); - if (countObj && countObj->type == cJSON_Number) { - count = countObj->valueint; - } else if (countObj && countObj->type != cJSON_Number) { - printf("failed to read json, column count not found"); - goto PARSE_OVER; - } else { - count = 1; - } - - // column info - memset(&columnCase, 0, sizeof(StrColumn)); - cJSON *dataType = cJSON_GetObjectItem(column, "type"); - if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { - printf("failed to read json, column type not found"); - goto PARSE_OVER; - } - //strncpy(superTbls->columns[k].dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - strncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - - cJSON* dataLen = cJSON_GetObjectItem(column, "len"); - if (dataLen && dataLen->type == cJSON_Number) { - columnCase.dataLen = dataLen->valueint; - } else if (dataLen && dataLen->type != cJSON_Number) { - printf("failed to read json, column len not found"); - goto PARSE_OVER; - } else { - columnCase.dataLen = 8; - } - - for (int n = 0; n < count; ++n) { - strncpy(superTbls->columns[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); - superTbls->columns[index].dataLen = columnCase.dataLen; - index++; - } - } - superTbls->columnCount = index; - - count = 1; - index = 0; - // tags - cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); - if (!tags || tags->type != cJSON_Array) { - printf("failed to read json, tags not found"); - goto PARSE_OVER; - } - - int tagSize = cJSON_GetArraySize(tags); - if (tagSize > MAX_TAG_COUNT) { - printf("failed to read json, tags size overflow, max tag size is %d\n", MAX_TAG_COUNT); - goto PARSE_OVER; - } - - //superTbls->tagCount = tagSize; - for (int k = 0; k < tagSize; ++k) { - cJSON* tag = cJSON_GetArrayItem(tags, k); - if (tag == NULL) continue; - - count = 1; - cJSON* countObj = cJSON_GetObjectItem(tag, "count"); - if (countObj && countObj->type == cJSON_Number) { - count = countObj->valueint; - } else if (countObj && countObj->type != cJSON_Number) { - printf("failed to read json, column count not found"); - goto PARSE_OVER; - } else { - count = 1; - } - - // column info - memset(&columnCase, 0, sizeof(StrColumn)); - cJSON *dataType = cJSON_GetObjectItem(tag, "type"); - if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { - printf("failed to read json, tag type not found"); - goto PARSE_OVER; - } - strncpy(columnCase.dataType, dataType->valuestring, MAX_TB_NAME_SIZE); - - cJSON* dataLen = cJSON_GetObjectItem(tag, "len"); - if (dataLen && dataLen->type == cJSON_Number) { - columnCase.dataLen = dataLen->valueint; - } else if (dataLen && dataLen->type != cJSON_Number) { - printf("failed to read json, column len not found"); - goto PARSE_OVER; - } else { - columnCase.dataLen = 0; - } - - for (int n = 0; n < count; ++n) { - strncpy(superTbls->tags[index].dataType, columnCase.dataType, MAX_TB_NAME_SIZE); - superTbls->tags[index].dataLen = columnCase.dataLen; - index++; - } - } - superTbls->tagCount = index; - - ret = true; - -PARSE_OVER: - //free(content); - //cJSON_Delete(root); - //fclose(fp); - return ret; -} - -static bool getMetaFromInsertJsonFile(cJSON* root) { - bool ret = false; - - cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); - if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { - strncpy(g_Dbs.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); - } - - cJSON* host = cJSON_GetObjectItem(root, "host"); - if (host && host->type == cJSON_String && host->valuestring != NULL) { - strncpy(g_Dbs.host, host->valuestring, MAX_DB_NAME_SIZE); - } else if (!host) { - strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); - } else { - printf("failed to read json, host not found\n"); - goto PARSE_OVER; - } - - cJSON* port = cJSON_GetObjectItem(root, "port"); - if (port && port->type == cJSON_Number) { - g_Dbs.port = port->valueint; - } else if (!port) { - g_Dbs.port = 6030; - } - - cJSON* user = cJSON_GetObjectItem(root, "user"); - if (user && user->type == cJSON_String && user->valuestring != NULL) { - strncpy(g_Dbs.user, user->valuestring, MAX_DB_NAME_SIZE); - } else if (!user) { - strncpy(g_Dbs.user, "root", MAX_DB_NAME_SIZE); - } - - cJSON* password = cJSON_GetObjectItem(root, "password"); - if (password && password->type == cJSON_String && password->valuestring != NULL) { - strncpy(g_Dbs.password, password->valuestring, MAX_DB_NAME_SIZE); - } else if (!password) { - strncpy(g_Dbs.password, "taosdata", MAX_DB_NAME_SIZE); - } - - cJSON* resultfile = cJSON_GetObjectItem(root, "result_file"); - if (resultfile && resultfile->type == cJSON_String && resultfile->valuestring != NULL) { - strncpy(g_Dbs.resultFile, resultfile->valuestring, MAX_FILE_NAME_LEN); - } else if (!resultfile) { - strncpy(g_Dbs.resultFile, "./insert_res.txt", MAX_FILE_NAME_LEN); - } - - cJSON* threads = cJSON_GetObjectItem(root, "thread_count"); - if (threads && threads->type == cJSON_Number) { - g_Dbs.threadCount = threads->valueint; - } else if (!threads) { - g_Dbs.threadCount = 1; - } else { - printf("failed to read json, threads not found"); - goto PARSE_OVER; - } - - cJSON* threads2 = cJSON_GetObjectItem(root, "thread_count_create_tbl"); - if (threads2 && threads2->type == cJSON_Number) { - g_Dbs.threadCountByCreateTbl = threads2->valueint; - } else if (!threads2) { - g_Dbs.threadCountByCreateTbl = 1; - } else { - printf("failed to read json, threads2 not found"); - goto PARSE_OVER; - } - - cJSON* dbs = cJSON_GetObjectItem(root, "databases"); - if (!dbs || dbs->type != cJSON_Array) { - printf("failed to read json, databases not found\n"); - goto PARSE_OVER; - } - - int dbSize = cJSON_GetArraySize(dbs); - if (dbSize > MAX_DB_COUNT) { - printf("failed to read json, databases size overflow, max database is %d\n", MAX_DB_COUNT); - goto PARSE_OVER; - } - - g_Dbs.dbCount = dbSize; - for (int i = 0; i < dbSize; ++i) { - cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); - if (dbinfos == NULL) continue; - - // dbinfo - cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); - if (!dbinfo || dbinfo->type != cJSON_Object) { - printf("failed to read json, dbinfo not found"); - goto PARSE_OVER; - } - - cJSON *dbName = cJSON_GetObjectItem(dbinfo, "name"); - if (!dbName || dbName->type != cJSON_String || dbName->valuestring == NULL) { - printf("failed to read json, db name not found"); - goto PARSE_OVER; - } - strncpy(g_Dbs.db[i].dbName, dbName->valuestring, MAX_DB_NAME_SIZE); - - cJSON *drop = cJSON_GetObjectItem(dbinfo, "drop"); - if (drop && drop->type == cJSON_String && drop->valuestring != NULL) { - if (0 == strncasecmp(drop->valuestring, "yes", 3)) { - g_Dbs.db[i].drop = 1; - } else { - g_Dbs.db[i].drop = 0; - } - } else if (!drop) { - g_Dbs.db[i].drop = 0; - } else { - printf("failed to read json, drop not found"); - goto PARSE_OVER; - } - - cJSON *precision = cJSON_GetObjectItem(dbinfo, "precision"); - if (precision && precision->type == cJSON_String && precision->valuestring != NULL) { - strncpy(g_Dbs.db[i].dbCfg.precision, precision->valuestring, MAX_DB_NAME_SIZE); - } else if (!precision) { - //strncpy(g_Dbs.db[i].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); - memset(g_Dbs.db[i].dbCfg.precision, 0, MAX_DB_NAME_SIZE); - } else { - printf("failed to read json, precision not found"); - goto PARSE_OVER; - } - - cJSON* update = cJSON_GetObjectItem(dbinfo, "update"); - if (update && update->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.update = update->valueint; - } else if (!update) { - g_Dbs.db[i].dbCfg.update = -1; - } else { - printf("failed to read json, update not found"); - goto PARSE_OVER; - } - - cJSON* replica = cJSON_GetObjectItem(dbinfo, "replica"); - if (replica && replica->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.replica = replica->valueint; - } else if (!replica) { - g_Dbs.db[i].dbCfg.replica = -1; - } else { - printf("failed to read json, replica not found"); - goto PARSE_OVER; - } - - cJSON* keep = cJSON_GetObjectItem(dbinfo, "keep"); - if (keep && keep->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.keep = keep->valueint; - } else if (!keep) { - g_Dbs.db[i].dbCfg.keep = -1; - } else { - printf("failed to read json, keep not found"); - goto PARSE_OVER; - } - - cJSON* days = cJSON_GetObjectItem(dbinfo, "days"); - if (days && days->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.days = days->valueint; - } else if (!days) { - g_Dbs.db[i].dbCfg.days = -1; - } else { - printf("failed to read json, days not found"); - goto PARSE_OVER; - } - - cJSON* cache = cJSON_GetObjectItem(dbinfo, "cache"); - if (cache && cache->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.cache = cache->valueint; - } else if (!cache) { - g_Dbs.db[i].dbCfg.cache = -1; - } else { - printf("failed to read json, cache not found"); - goto PARSE_OVER; - } - - cJSON* blocks= cJSON_GetObjectItem(dbinfo, "blocks"); - if (blocks && blocks->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.blocks = blocks->valueint; - } else if (!blocks) { - g_Dbs.db[i].dbCfg.blocks = -1; - } else { - printf("failed to read json, block not found"); - goto PARSE_OVER; - } - - //cJSON* maxtablesPerVnode= cJSON_GetObjectItem(dbinfo, "maxtablesPerVnode"); - //if (maxtablesPerVnode && maxtablesPerVnode->type == cJSON_Number) { - // g_Dbs.db[i].dbCfg.maxtablesPerVnode = maxtablesPerVnode->valueint; - //} else if (!maxtablesPerVnode) { - // g_Dbs.db[i].dbCfg.maxtablesPerVnode = TSDB_DEFAULT_TABLES; - //} else { - // printf("failed to read json, maxtablesPerVnode not found"); - // goto PARSE_OVER; - //} - - cJSON* minRows= cJSON_GetObjectItem(dbinfo, "minRows"); - if (minRows && minRows->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.minRows = minRows->valueint; - } else if (!minRows) { - g_Dbs.db[i].dbCfg.minRows = -1; - } else { - printf("failed to read json, minRows not found"); - goto PARSE_OVER; - } - - cJSON* maxRows= cJSON_GetObjectItem(dbinfo, "maxRows"); - if (maxRows && maxRows->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.maxRows = maxRows->valueint; - } else if (!maxRows) { - g_Dbs.db[i].dbCfg.maxRows = -1; - } else { - printf("failed to read json, maxRows not found"); - goto PARSE_OVER; - } - - cJSON* comp= cJSON_GetObjectItem(dbinfo, "comp"); - if (comp && comp->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.comp = comp->valueint; - } else if (!comp) { - g_Dbs.db[i].dbCfg.comp = -1; - } else { - printf("failed to read json, comp not found"); - goto PARSE_OVER; - } - - cJSON* walLevel= cJSON_GetObjectItem(dbinfo, "walLevel"); - if (walLevel && walLevel->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.walLevel = walLevel->valueint; - } else if (!walLevel) { - g_Dbs.db[i].dbCfg.walLevel = -1; - } else { - printf("failed to read json, walLevel not found"); - goto PARSE_OVER; - } - - cJSON* quorum= cJSON_GetObjectItem(dbinfo, "quorum"); - if (quorum && quorum->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.quorum = quorum->valueint; - } else if (!quorum) { - g_Dbs.db[i].dbCfg.quorum = -1; - } else { - printf("failed to read json, walLevel not found"); - goto PARSE_OVER; - } - - cJSON* fsync= cJSON_GetObjectItem(dbinfo, "fsync"); - if (fsync && fsync->type == cJSON_Number) { - g_Dbs.db[i].dbCfg.fsync = fsync->valueint; - } else if (!fsync) { - g_Dbs.db[i].dbCfg.fsync = -1; - } else { - printf("failed to read json, fsync not found"); - goto PARSE_OVER; - } - - // super_talbes - cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); - if (!stables || stables->type != cJSON_Array) { - printf("failed to read json, super_tables not found"); - goto PARSE_OVER; - } - - int stbSize = cJSON_GetArraySize(stables); - if (stbSize > MAX_SUPER_TABLE_COUNT) { - printf("failed to read json, databases size overflow, max database is %d\n", MAX_SUPER_TABLE_COUNT); - goto PARSE_OVER; - } - - g_Dbs.db[i].superTblCount = stbSize; - for (int j = 0; j < stbSize; ++j) { - cJSON* stbInfo = cJSON_GetArrayItem(stables, j); - if (stbInfo == NULL) continue; - - // dbinfo - cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); - if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { - printf("failed to read json, stb name not found"); - goto PARSE_OVER; - } - strncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); - - cJSON *prefix = cJSON_GetObjectItem(stbInfo, "childtable_prefix"); - if (!prefix || prefix->type != cJSON_String || prefix->valuestring == NULL) { - printf("failed to read json, childtable_prefix not found"); - goto PARSE_OVER; - } - strncpy(g_Dbs.db[i].superTbls[j].childTblPrefix, prefix->valuestring, MAX_DB_NAME_SIZE); - - cJSON *autoCreateTbl = cJSON_GetObjectItem(stbInfo, "auto_create_table"); // yes, no, null - if (autoCreateTbl && autoCreateTbl->type == cJSON_String && autoCreateTbl->valuestring != NULL) { - if (0 == strncasecmp(autoCreateTbl->valuestring, "yes", 3)) { - g_Dbs.db[i].superTbls[j].autoCreateTable = AUTO_CREATE_SUBTBL; - } else if (0 == strncasecmp(autoCreateTbl->valuestring, "no", 2)) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } else { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } - } else if (!autoCreateTbl) { - g_Dbs.db[i].superTbls[j].autoCreateTable = PRE_CREATE_SUBTBL; - } else { - printf("failed to read json, auto_create_table not found"); - goto PARSE_OVER; - } - - cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no - if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) { - if (0 == strncasecmp(childTblExists->valuestring, "yes", 3)) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_ALREADY_EXISTS; - } else if (0 == strncasecmp(childTblExists->valuestring, "no", 2)) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } else { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } - } else if (!childTblExists) { - g_Dbs.db[i].superTbls[j].childTblExists = TBL_NO_EXISTS; - } else { - printf("failed to read json, child_table_exists not found"); - goto PARSE_OVER; - } - - cJSON* count = cJSON_GetObjectItem(stbInfo, "childtable_count"); - if (!count || count->type != cJSON_Number || 0 >= count->valueint) { - printf("failed to read json, childtable_count not found"); - goto PARSE_OVER; - } - g_Dbs.db[i].superTbls[j].childTblCount = count->valueint; - - cJSON *dataSource = cJSON_GetObjectItem(stbInfo, "data_source"); - if (dataSource && dataSource->type == cJSON_String && dataSource->valuestring != NULL) { - strncpy(g_Dbs.db[i].superTbls[j].dataSource, dataSource->valuestring, MAX_DB_NAME_SIZE); - } else if (!dataSource) { - strncpy(g_Dbs.db[i].superTbls[j].dataSource, "rand", MAX_DB_NAME_SIZE); - } else { - printf("failed to read json, data_source not found"); - goto PARSE_OVER; - } - - cJSON *insertMode = cJSON_GetObjectItem(stbInfo, "insert_mode"); // taosc , restful - if (insertMode && insertMode->type == cJSON_String && insertMode->valuestring != NULL) { - strncpy(g_Dbs.db[i].superTbls[j].insertMode, insertMode->valuestring, MAX_DB_NAME_SIZE); - #ifndef TD_LOWA_CURL - if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 7)) { - printf("There no libcurl, so no support resetful test! please use taosc mode.\n"); - goto PARSE_OVER; - } - #endif - } else if (!insertMode) { - strncpy(g_Dbs.db[i].superTbls[j].insertMode, "taosc", MAX_DB_NAME_SIZE); - } else { - printf("failed to read json, insert_mode not found"); - goto PARSE_OVER; - } - - cJSON *ts = cJSON_GetObjectItem(stbInfo, "start_timestamp"); - if (ts && ts->type == cJSON_String && ts->valuestring != NULL) { - strncpy(g_Dbs.db[i].superTbls[j].startTimestamp, ts->valuestring, MAX_DB_NAME_SIZE); - } else if (!ts) { - strncpy(g_Dbs.db[i].superTbls[j].startTimestamp, "now", MAX_DB_NAME_SIZE); - } else { - printf("failed to read json, start_timestamp not found"); - goto PARSE_OVER; - } - - cJSON* timestampStep = cJSON_GetObjectItem(stbInfo, "timestamp_step"); - if (timestampStep && timestampStep->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].timeStampStep = timestampStep->valueint; - } else if (!timestampStep) { - g_Dbs.db[i].superTbls[j].timeStampStep = 1000; - } else { - printf("failed to read json, timestamp_step not found"); - goto PARSE_OVER; - } - - cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size"); - if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint; - if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) { - g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; - } - } else if (!sampleDataBufSize) { - g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; - } else { - printf("failed to read json, sample_buf_size not found"); - goto PARSE_OVER; - } - - cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); - if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { - strncpy(g_Dbs.db[i].superTbls[j].sampleFormat, sampleFormat->valuestring, MAX_DB_NAME_SIZE); - } else if (!sampleFormat) { - strncpy(g_Dbs.db[i].superTbls[j].sampleFormat, "csv", MAX_DB_NAME_SIZE); - } else { - printf("failed to read json, sample_format not found"); - goto PARSE_OVER; - } - - cJSON *sampleFile = cJSON_GetObjectItem(stbInfo, "sample_file"); - if (sampleFile && sampleFile->type == cJSON_String && sampleFile->valuestring != NULL) { - strncpy(g_Dbs.db[i].superTbls[j].sampleFile, sampleFile->valuestring, MAX_FILE_NAME_LEN); - } else if (!sampleFile) { - memset(g_Dbs.db[i].superTbls[j].sampleFile, 0, MAX_FILE_NAME_LEN); - } else { - printf("failed to read json, sample_file not found"); - goto PARSE_OVER; - } - - cJSON *tagsFile = cJSON_GetObjectItem(stbInfo, "tags_file"); - if (tagsFile && tagsFile->type == cJSON_String && tagsFile->valuestring != NULL) { - strncpy(g_Dbs.db[i].superTbls[j].tagsFile, tagsFile->valuestring, MAX_FILE_NAME_LEN); - if (0 == g_Dbs.db[i].superTbls[j].tagsFile[0]) { - g_Dbs.db[i].superTbls[j].tagSource = 0; - } else { - g_Dbs.db[i].superTbls[j].tagSource = 1; - } - } else if (!tagsFile) { - memset(g_Dbs.db[i].superTbls[j].tagsFile, 0, MAX_FILE_NAME_LEN); - g_Dbs.db[i].superTbls[j].tagSource = 0; - } else { - printf("failed to read json, tags_file not found"); - goto PARSE_OVER; - } - - cJSON* maxSqlLen = cJSON_GetObjectItem(stbInfo, "max_sql_len"); - if (maxSqlLen && maxSqlLen->type == cJSON_Number) { - int32_t len = maxSqlLen->valueint; - if (len > TSDB_MAX_ALLOWED_SQL_LEN) { - len = TSDB_MAX_ALLOWED_SQL_LEN; - } else if (len < TSDB_MAX_SQL_LEN) { - len = TSDB_MAX_SQL_LEN; - } - g_Dbs.db[i].superTbls[j].maxSqlLen = len; - } else if (!maxSqlLen) { - g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN; - } else { - printf("failed to read json, maxSqlLen not found"); - goto PARSE_OVER; - } - - cJSON *multiThreadWriteOneTbl = cJSON_GetObjectItem(stbInfo, "multi_thread_write_one_tbl"); // no , yes - if (multiThreadWriteOneTbl && multiThreadWriteOneTbl->type == cJSON_String && multiThreadWriteOneTbl->valuestring != NULL) { - if (0 == strncasecmp(multiThreadWriteOneTbl->valuestring, "yes", 3)) { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 1; - } else { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; - } - } else if (!multiThreadWriteOneTbl) { - g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl = 0; - } else { - printf("failed to read json, multiThreadWriteOneTbl not found"); - goto PARSE_OVER; - } - - cJSON* numberOfTblInOneSql = cJSON_GetObjectItem(stbInfo, "number_of_tbl_in_one_sql"); - if (numberOfTblInOneSql && numberOfTblInOneSql->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = numberOfTblInOneSql->valueint; - } else if (!numberOfTblInOneSql) { - g_Dbs.db[i].superTbls[j].numberOfTblInOneSql = 0; - } else { - printf("failed to read json, numberOfTblInOneSql not found"); - goto PARSE_OVER; - } - - cJSON* rowsPerTbl = cJSON_GetObjectItem(stbInfo, "rows_per_tbl"); - if (rowsPerTbl && rowsPerTbl->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].rowsPerTbl = rowsPerTbl->valueint; - } else if (!rowsPerTbl) { - g_Dbs.db[i].superTbls[j].rowsPerTbl = 1; - } else { - printf("failed to read json, rowsPerTbl not found"); - goto PARSE_OVER; - } - - cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); - if (disorderRatio && disorderRatio->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; - } else if (!disorderRatio) { - g_Dbs.db[i].superTbls[j].disorderRatio = 0; - } else { - printf("failed to read json, disorderRatio not found"); - goto PARSE_OVER; - } - - cJSON* disorderRange = cJSON_GetObjectItem(stbInfo, "disorder_range"); - if (disorderRange && disorderRange->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].disorderRange = disorderRange->valueint; - } else if (!disorderRange) { - g_Dbs.db[i].superTbls[j].disorderRange = 1000; - } else { - printf("failed to read json, disorderRange not found"); - goto PARSE_OVER; - } - - cJSON* insertRate = cJSON_GetObjectItem(stbInfo, "insert_rate"); - if (insertRate && insertRate->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].insertRate = insertRate->valueint; - } else if (!insertRate) { - g_Dbs.db[i].superTbls[j].insertRate = 0; - } else { - printf("failed to read json, insert_rate not found"); - goto PARSE_OVER; - } - - cJSON* insertRows = cJSON_GetObjectItem(stbInfo, "insert_rows"); - if (insertRows && insertRows->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].insertRows = insertRows->valueint; - if (0 == g_Dbs.db[i].superTbls[j].insertRows) { - g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; - } - } else if (!insertRows) { - g_Dbs.db[i].superTbls[j].insertRows = 0x7FFFFFFFFFFFFFFF; - } else { - printf("failed to read json, insert_rows not found"); - goto PARSE_OVER; - } - - if (NO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { - continue; - } - - int retVal = getColumnAndTagTypeFromInsertJsonFile(stbInfo, &g_Dbs.db[i].superTbls[j]); - if (false == retVal) { - goto PARSE_OVER; - } - } - } - - ret = true; - -PARSE_OVER: - //free(content); - //cJSON_Delete(root); - //fclose(fp); - return ret; -} - -static bool getMetaFromQueryJsonFile(cJSON* root) { - bool ret = false; - - cJSON* cfgdir = cJSON_GetObjectItem(root, "cfgdir"); - if (cfgdir && cfgdir->type == cJSON_String && cfgdir->valuestring != NULL) { - strncpy(g_queryInfo.cfgDir, cfgdir->valuestring, MAX_FILE_NAME_LEN); - } - - cJSON* host = cJSON_GetObjectItem(root, "host"); - if (host && host->type == cJSON_String && host->valuestring != NULL) { - strncpy(g_queryInfo.host, host->valuestring, MAX_DB_NAME_SIZE); - } else if (!host) { - strncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); - } else { - printf("failed to read json, host not found\n"); - goto PARSE_OVER; - } - - cJSON* port = cJSON_GetObjectItem(root, "port"); - if (port && port->type == cJSON_Number) { - g_queryInfo.port = port->valueint; - } else if (!port) { - g_queryInfo.port = 6030; - } - - cJSON* user = cJSON_GetObjectItem(root, "user"); - if (user && user->type == cJSON_String && user->valuestring != NULL) { - strncpy(g_queryInfo.user, user->valuestring, MAX_DB_NAME_SIZE); - } else if (!user) { - strncpy(g_queryInfo.user, "root", MAX_DB_NAME_SIZE); ; - } - - cJSON* password = cJSON_GetObjectItem(root, "password"); - if (password && password->type == cJSON_String && password->valuestring != NULL) { - strncpy(g_queryInfo.password, password->valuestring, MAX_DB_NAME_SIZE); - } else if (!password) { - strncpy(g_queryInfo.password, "taosdata", MAX_DB_NAME_SIZE);; - } - - cJSON* dbs = cJSON_GetObjectItem(root, "databases"); - if (dbs && dbs->type == cJSON_String && dbs->valuestring != NULL) { - strncpy(g_queryInfo.dbName, dbs->valuestring, MAX_DB_NAME_SIZE); - } else if (!dbs) { - printf("failed to read json, databases not found\n"); - goto PARSE_OVER; - } - - cJSON* queryMode = cJSON_GetObjectItem(root, "query_mode"); - if (queryMode && queryMode->type == cJSON_String && queryMode->valuestring != NULL) { - strncpy(g_queryInfo.queryMode, queryMode->valuestring, MAX_TB_NAME_SIZE); - } else if (!queryMode) { - strncpy(g_queryInfo.queryMode, "taosc", MAX_TB_NAME_SIZE); - } else { - printf("failed to read json, query_mode not found\n"); - goto PARSE_OVER; - } - - // super_table_query - cJSON *superQuery = cJSON_GetObjectItem(root, "specified_table_query"); - if (!superQuery) { - g_queryInfo.superQueryInfo.concurrent = 0; - g_queryInfo.superQueryInfo.sqlCount = 0; - } else if (superQuery->type != cJSON_Object) { - printf("failed to read json, super_table_query not found"); - goto PARSE_OVER; - } else { - cJSON* rate = cJSON_GetObjectItem(superQuery, "query_interval"); - if (rate && rate->type == cJSON_Number) { - g_queryInfo.superQueryInfo.rate = rate->valueint; - } else if (!rate) { - g_queryInfo.superQueryInfo.rate = 0; - } - - cJSON* concurrent = cJSON_GetObjectItem(superQuery, "concurrent"); - if (concurrent && concurrent->type == cJSON_Number) { - g_queryInfo.superQueryInfo.concurrent = concurrent->valueint; - } else if (!concurrent) { - g_queryInfo.superQueryInfo.concurrent = 1; - } - - cJSON* mode = cJSON_GetObjectItem(superQuery, "mode"); - if (mode && mode->type == cJSON_String && mode->valuestring != NULL) { - if (0 == strcmp("sync", mode->valuestring)) { - g_queryInfo.superQueryInfo.subscribeMode = 0; - } else if (0 == strcmp("async", mode->valuestring)) { - g_queryInfo.superQueryInfo.subscribeMode = 1; - } else { - printf("failed to read json, subscribe mod error\n"); - goto PARSE_OVER; - } - } else { - g_queryInfo.superQueryInfo.subscribeMode = 0; - } - - cJSON* interval = cJSON_GetObjectItem(superQuery, "interval"); - if (interval && interval->type == cJSON_Number) { - g_queryInfo.superQueryInfo.subscribeInterval = interval->valueint; - } else if (!interval) { - //printf("failed to read json, subscribe interval no found\n"); - //goto PARSE_OVER; - g_queryInfo.superQueryInfo.subscribeInterval = 10000; - } - - cJSON* restart = cJSON_GetObjectItem(superQuery, "restart"); - if (restart && restart->type == cJSON_String && restart->valuestring != NULL) { - if (0 == strcmp("yes", restart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 1; - } else if (0 == strcmp("no", restart->valuestring)) { - g_queryInfo.superQueryInfo.subscribeRestart = 0; - } else { - printf("failed to read json, subscribe restart error\n"); - goto PARSE_OVER; - } - } else { - g_queryInfo.superQueryInfo.subscribeRestart = 1; - } - - cJSON* keepProgress = cJSON_GetObjectItem(superQuery, "keepProgress"); - if (keepProgress && keepProgress->type == cJSON_String && keepProgress->valuestring != NULL) { - if (0 == strcmp("yes", keepProgress->valuestring)) { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", keepProgress->valuestring)) { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; - } else { - printf("failed to read json, subscribe keepProgress error\n"); - goto PARSE_OVER; - } - } else { - g_queryInfo.superQueryInfo.subscribeKeepProgress = 0; - } - - // sqls - cJSON* superSqls = cJSON_GetObjectItem(superQuery, "sqls"); - if (!superSqls) { - g_queryInfo.superQueryInfo.sqlCount = 0; - } else if (superSqls->type != cJSON_Array) { - printf("failed to read json, super sqls not found\n"); - goto PARSE_OVER; - } else { - int superSqlSize = cJSON_GetArraySize(superSqls); - if (superSqlSize > MAX_QUERY_SQL_COUNT) { - printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); - goto PARSE_OVER; - } - - g_queryInfo.superQueryInfo.sqlCount = superSqlSize; - for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(superSqls, j); - if (sql == NULL) continue; - - cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); - if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { - printf("failed to read json, sql not found\n"); - goto PARSE_OVER; - } - strncpy(g_queryInfo.superQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); - - cJSON *result = cJSON_GetObjectItem(sql, "result"); - if (NULL != result && result->type == cJSON_String && result->valuestring != NULL) { - strncpy(g_queryInfo.superQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); - } else if (NULL == result) { - memset(g_queryInfo.superQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); - } else { - printf("failed to read json, super query result file not found\n"); - goto PARSE_OVER; - } - } - } - } - - // sub_table_query - cJSON *subQuery = cJSON_GetObjectItem(root, "super_table_query"); - if (!subQuery) { - g_queryInfo.subQueryInfo.threadCnt = 0; - g_queryInfo.subQueryInfo.sqlCount = 0; - } else if (subQuery->type != cJSON_Object) { - printf("failed to read json, sub_table_query not found"); - ret = true; - goto PARSE_OVER; - } else { - cJSON* subrate = cJSON_GetObjectItem(subQuery, "query_interval"); - if (subrate && subrate->type == cJSON_Number) { - g_queryInfo.subQueryInfo.rate = subrate->valueint; - } else if (!subrate) { - g_queryInfo.subQueryInfo.rate = 0; - } - - cJSON* threads = cJSON_GetObjectItem(subQuery, "threads"); - if (threads && threads->type == cJSON_Number) { - g_queryInfo.subQueryInfo.threadCnt = threads->valueint; - } else if (!threads) { - g_queryInfo.subQueryInfo.threadCnt = 1; - } - - //cJSON* subTblCnt = cJSON_GetObjectItem(subQuery, "childtable_count"); - //if (subTblCnt && subTblCnt->type == cJSON_Number) { - // g_queryInfo.subQueryInfo.childTblCount = subTblCnt->valueint; - //} else if (!subTblCnt) { - // g_queryInfo.subQueryInfo.childTblCount = 0; - //} - - cJSON* stblname = cJSON_GetObjectItem(subQuery, "stblname"); - if (stblname && stblname->type == cJSON_String && stblname->valuestring != NULL) { - strncpy(g_queryInfo.subQueryInfo.sTblName, stblname->valuestring, MAX_TB_NAME_SIZE); - } else { - printf("failed to read json, super table name not found\n"); - goto PARSE_OVER; - } - - cJSON* submode = cJSON_GetObjectItem(subQuery, "mode"); - if (submode && submode->type == cJSON_String && submode->valuestring != NULL) { - if (0 == strcmp("sync", submode->valuestring)) { - g_queryInfo.subQueryInfo.subscribeMode = 0; - } else if (0 == strcmp("async", submode->valuestring)) { - g_queryInfo.subQueryInfo.subscribeMode = 1; - } else { - printf("failed to read json, subscribe mod error\n"); - goto PARSE_OVER; - } - } else { - g_queryInfo.subQueryInfo.subscribeMode = 0; - } - - cJSON* subinterval = cJSON_GetObjectItem(subQuery, "interval"); - if (subinterval && subinterval->type == cJSON_Number) { - g_queryInfo.subQueryInfo.subscribeInterval = subinterval->valueint; - } else if (!subinterval) { - //printf("failed to read json, subscribe interval no found\n"); - //goto PARSE_OVER; - g_queryInfo.subQueryInfo.subscribeInterval = 10000; - } - - cJSON* subrestart = cJSON_GetObjectItem(subQuery, "restart"); - if (subrestart && subrestart->type == cJSON_String && subrestart->valuestring != NULL) { - if (0 == strcmp("yes", subrestart->valuestring)) { - g_queryInfo.subQueryInfo.subscribeRestart = 1; - } else if (0 == strcmp("no", subrestart->valuestring)) { - g_queryInfo.subQueryInfo.subscribeRestart = 0; - } else { - printf("failed to read json, subscribe restart error\n"); - goto PARSE_OVER; - } - } else { - g_queryInfo.subQueryInfo.subscribeRestart = 1; - } - - cJSON* subkeepProgress = cJSON_GetObjectItem(subQuery, "keepProgress"); - if (subkeepProgress && subkeepProgress->type == cJSON_String && subkeepProgress->valuestring != NULL) { - if (0 == strcmp("yes", subkeepProgress->valuestring)) { - g_queryInfo.subQueryInfo.subscribeKeepProgress = 1; - } else if (0 == strcmp("no", subkeepProgress->valuestring)) { - g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; - } else { - printf("failed to read json, subscribe keepProgress error\n"); - goto PARSE_OVER; - } - } else { - g_queryInfo.subQueryInfo.subscribeKeepProgress = 0; - } - - // sqls - cJSON* subsqls = cJSON_GetObjectItem(subQuery, "sqls"); - if (!subsqls) { - g_queryInfo.subQueryInfo.sqlCount = 0; - } else if (subsqls->type != cJSON_Array) { - printf("failed to read json, super sqls not found\n"); - goto PARSE_OVER; - } else { - int superSqlSize = cJSON_GetArraySize(subsqls); - if (superSqlSize > MAX_QUERY_SQL_COUNT) { - printf("failed to read json, query sql size overflow, max is %d\n", MAX_QUERY_SQL_COUNT); - goto PARSE_OVER; - } - - g_queryInfo.subQueryInfo.sqlCount = superSqlSize; - for (int j = 0; j < superSqlSize; ++j) { - cJSON* sql = cJSON_GetArrayItem(subsqls, j); - if (sql == NULL) continue; - - cJSON *sqlStr = cJSON_GetObjectItem(sql, "sql"); - if (!sqlStr || sqlStr->type != cJSON_String || sqlStr->valuestring == NULL) { - printf("failed to read json, sql not found\n"); - goto PARSE_OVER; - } - strncpy(g_queryInfo.subQueryInfo.sql[j], sqlStr->valuestring, MAX_QUERY_SQL_LENGTH); - - cJSON *result = cJSON_GetObjectItem(sql, "result"); - if (result != NULL && result->type == cJSON_String && result->valuestring != NULL){ - strncpy(g_queryInfo.subQueryInfo.result[j], result->valuestring, MAX_FILE_NAME_LEN); - } else if (NULL == result) { - memset(g_queryInfo.subQueryInfo.result[j], 0, MAX_FILE_NAME_LEN); - } else { - printf("failed to read json, sub query result file not found\n"); - goto PARSE_OVER; - } - } - } - } - - ret = true; - -PARSE_OVER: - //free(content); - //cJSON_Delete(root); - //fclose(fp); - return ret; -} - -static bool getInfoFromJsonFile(char* file) { - FILE *fp = fopen(file, "r"); - if (!fp) { - printf("failed to read %s, reason:%s\n", file, strerror(errno)); - return false; - } - - bool ret = false; - int maxLen = 64000; - char *content = calloc(1, maxLen + 1); - int len = fread(content, 1, maxLen, fp); - if (len <= 0) { - free(content); - fclose(fp); - printf("failed to read %s, content is null", file); - return false; - } - - content[len] = 0; - cJSON* root = cJSON_Parse(content); - if (root == NULL) { - printf("failed to cjson parse %s, invalid json format", file); - goto PARSE_OVER; - } - - cJSON* filetype = cJSON_GetObjectItem(root, "filetype"); - if (filetype && filetype->type == cJSON_String && filetype->valuestring != NULL) { - if (0 == strcasecmp("insert", filetype->valuestring)) { - g_jsonType = INSERT_MODE; - } else if (0 == strcasecmp("query", filetype->valuestring)) { - g_jsonType = QUERY_MODE; - } else if (0 == strcasecmp("subscribe", filetype->valuestring)) { - g_jsonType = SUBSCRIBE_MODE; - } else { - printf("failed to read json, filetype not support\n"); - goto PARSE_OVER; - } - } else if (!filetype) { - g_jsonType = INSERT_MODE; - } else { - printf("failed to read json, filetype not found\n"); - goto PARSE_OVER; - } - - if (INSERT_MODE == g_jsonType) { - ret = getMetaFromInsertJsonFile(root); - } else if (QUERY_MODE == g_jsonType) { - ret = getMetaFromQueryJsonFile(root); - } else if (SUBSCRIBE_MODE == g_jsonType) { - ret = getMetaFromQueryJsonFile(root); - } else { - printf("input json file type error! please input correct file type: insert or query or subscribe\n"); - goto PARSE_OVER; - } - -PARSE_OVER: - free(content); - cJSON_Delete(root); - fclose(fp); - return ret; -} - - -void prePareSampleData() { - for (int i = 0; i < g_Dbs.dbCount; i++) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - //if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].dataSource, "sample", 6)) { - // readSampleFromFileToMem(&g_Dbs.db[i].superTbls[j]); - //} - - if (g_Dbs.db[i].superTbls[j].tagsFile[0] != 0) { - (void)readTagFromCsvFileToMem(&g_Dbs.db[i].superTbls[j]); - } - - #ifdef TD_LOWA_CURL - if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 8)) { - curl_global_init(CURL_GLOBAL_ALL); - } - #endif - } - } -} - -void postFreeResource() { - tmfclose(g_fpOfInsertResult); - for (int i = 0; i < g_Dbs.dbCount; i++) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if (0 != g_Dbs.db[i].superTbls[j].colsOfCreatChildTable) { - free(g_Dbs.db[i].superTbls[j].colsOfCreatChildTable); - g_Dbs.db[i].superTbls[j].colsOfCreatChildTable = NULL; - } - if (0 != g_Dbs.db[i].superTbls[j].sampleDataBuf) { - free(g_Dbs.db[i].superTbls[j].sampleDataBuf); - g_Dbs.db[i].superTbls[j].sampleDataBuf = NULL; - } - if (0 != g_Dbs.db[i].superTbls[j].tagDataBuf) { - free(g_Dbs.db[i].superTbls[j].tagDataBuf); - g_Dbs.db[i].superTbls[j].tagDataBuf = NULL; - } - if (0 != g_Dbs.db[i].superTbls[j].childTblName) { - free(g_Dbs.db[i].superTbls[j].childTblName); - g_Dbs.db[i].superTbls[j].childTblName = NULL; - } - - #ifdef TD_LOWA_CURL - if (0 == strncasecmp(g_Dbs.db[i].superTbls[j].insertMode, "restful", 8)) { - curl_global_cleanup(); - } - #endif - } - } -} - -int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* superTblInfo, int* sampleUsePos, FILE *fp, char* sampleBuf) { - if ((*sampleUsePos) == MAX_SAMPLES_ONCE_FROM_FILE) { - int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleBuf); - if (0 != ret) { - return -1; - } - *sampleUsePos = 0; - } - - int dataLen = 0; - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%s", sampleBuf + superTblInfo->lenOfOneRow * (*sampleUsePos)); - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); - - (*sampleUsePos)++; - - return dataLen; -} - -int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) { - int dataLen = 0; - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); - for (int i = 0; i < stbInfo->columnCount; i++) { - if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) { - if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { - printf("binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); - return (-1); - } - - char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1); - if (NULL == buf) { - printf("calloc failed! size:%d\n", stbInfo->columns[i].dataLen); - return (-1); - } - rand_string(buf, stbInfo->columns[i].dataLen); - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf); - tmfree(buf); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "int", 3)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_int()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bigint", 6)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "float", 5)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%f, ", rand_float()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "double", 6)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%f, ", rand_double()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "smallint", 8)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool()); - } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); - } else { - printf("No support data type: %s\n", stbInfo->columns[i].dataType); - return (-1); - } - } - dataLen -= 2; - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); - - return dataLen; -} - -void syncWriteForNumberOfTblInOneSql(threadInfo *winfo, FILE *fp, char* sampleDataBuf) { - SSuperTable* superTblInfo = winfo->superTblInfo; - - int samplePos = 0; - - //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); - int64_t totalRowsInserted = 0; - int64_t totalAffectedRows = 0; - int64_t lastPrintTime = taosGetTimestampMs(); - - char* buffer = calloc(superTblInfo->maxSqlLen+1, 1); - if (NULL == buffer) { - printf("========calloc size[ %d ] fail!\n", superTblInfo->maxSqlLen); - return; - } - - int32_t numberOfTblInOneSql = superTblInfo->numberOfTblInOneSql; - int32_t tbls = winfo->end_table_id - winfo->start_table_id + 1; - if (numberOfTblInOneSql > tbls) { - numberOfTblInOneSql = tbls; - } - - int64_t time_counter = winfo->start_time; - int64_t tmp_time; - int sampleUsePos; - - int64_t st = 0; - int64_t et = 0; - for (int i = 0; i < superTblInfo->insertRows;) { - if (superTblInfo->insertRate && (et - st) < 1000) { - taosMsleep(1000 - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); - } - - if (superTblInfo->insertRate) { - st = taosGetTimestampMs(); - } - - int32_t tbl_id = 0; - for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; ) { - int inserted = i; - - int k = 0; - int batchRowsSql = 0; - while (1) - { - int len = 0; - memset(buffer, 0, superTblInfo->maxSqlLen); - char *pstr = buffer; - - int32_t end_tbl_id = tID + numberOfTblInOneSql; - if (end_tbl_id > winfo->end_table_id) { - end_tbl_id = winfo->end_table_id+1; - } - for (tbl_id = tID; tbl_id < end_tbl_id; tbl_id++) { - sampleUsePos = samplePos; - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { - char* tagsValBuf = NULL; - if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo); - } else { - tagsValBuf = getTagValueFromTagSample(superTblInfo, tbl_id % superTblInfo->tagSampleCount); - } - if (NULL == tagsValBuf) { - goto free_and_statistics; - } - - if (0 == len) { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d using %s.%s tags %s values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id, winfo->db_name, superTblInfo->sTblName, tagsValBuf); - } else { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s%d using %s.%s tags %s values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id, winfo->db_name, superTblInfo->sTblName, tagsValBuf); - } - tmfree(tagsValBuf); - } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { - if (0 == len) { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s values ", winfo->db_name, superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); - } else { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s values ", winfo->db_name, superTblInfo->childTblName + tbl_id * TSDB_TABLE_NAME_LEN); - } - } else { // pre-create child table - if (0 == len) { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id); - } else { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, " %s.%s%d values ", winfo->db_name, superTblInfo->childTblPrefix, tbl_id); - } - } - - tmp_time = time_counter; - for (k = 0; k < superTblInfo->rowsPerTbl;) { - int retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { - retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo, &sampleUsePos, fp, sampleDataBuf); - if (retLen < 0) { - goto free_and_statistics; - } - } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", 8)) { - int rand_num = rand_tinyint() % 100; - if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { - int64_t d = tmp_time - rand() % superTblInfo->disorderRange; - retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); - } else { - retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo); - } - if (retLen < 0) { - goto free_and_statistics; - } - } - len += retLen; - //inserted++; - k++; - totalRowsInserted++; - batchRowsSql++; - - if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128) || batchRowsSql >= INT16_MAX - 1) { - tID = tbl_id + 1; - printf("config rowsPerTbl and numberOfTblInOneSql not match with max_sql_lenth, please reconfig![lenOfOneRow:%d]\n", superTblInfo->lenOfOneRow); - goto send_to_server; - } - } - - } - - tID = tbl_id; - inserted += superTblInfo->rowsPerTbl; - - send_to_server: - batchRowsSql = 0; - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { - //printf("multi table===== sql: %s \n\n", buffer); - //int64_t t1 = taosGetTimestampMs(); - int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); - if (0 > affectedRows) { - goto free_and_statistics; - } - totalAffectedRows += affectedRows; - - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, totalRowsInserted, totalAffectedRows); - lastPrintTime = currentPrintTime; - } - //int64_t t2 = taosGetTimestampMs(); - //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); - } else { - #ifdef TD_LOWA_CURL - //int64_t t1 = taosGetTimestampMs(); - int retCode = curlProceSql(g_Dbs.host, g_Dbs.port, buffer, winfo->curl_handle); - //int64_t t2 = taosGetTimestampMs(); - //printf("http insert sql return, Spent %ld ms \n", t2 - t1); - - if (0 != retCode) { - printf("========curl return fail, threadID[%d]\n", winfo->threadID); - goto free_and_statistics; - } - #else - printf("========no use http mode for no curl lib!\n"); - goto free_and_statistics; - #endif - } - - //printf("========tID:%d, k:%d, loop_cnt:%d\n", tID, k, loop_cnt); - break; - } - - if (tID > winfo->end_table_id) { - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { - samplePos = sampleUsePos; - } - i = inserted; - time_counter = tmp_time; - } - } - - if (superTblInfo->insertRate) { - et = taosGetTimestampMs(); - } - //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); - } - - free_and_statistics: - tmfree(buffer); - winfo->totalRowsInserted = totalRowsInserted; - winfo->totalAffectedRows = totalAffectedRows; - printf("====thread[%d] completed total inserted rows: %"PRId64 ", affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); - return; -} - -// sync insertion -/* - 1 thread: 100 tables * 2000 rows/s - 1 thread: 10 tables * 20000 rows/s - 6 thread: 300 tables * 2000 rows/s - - 2 taosinsertdata , 1 thread: 10 tables * 20000 rows/s -*/ -void *syncWrite(void *sarg) { - int64_t totalRowsInserted = 0; - int64_t totalAffectedRows = 0; - int64_t lastPrintTime = taosGetTimestampMs(); - - threadInfo *winfo = (threadInfo *)sarg; - SSuperTable* superTblInfo = winfo->superTblInfo; - - FILE *fp = NULL; - char* sampleDataBuf = NULL; - int samplePos = 0; - - // each thread read sample data from csv file - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { - sampleDataBuf = calloc(superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); - if (sampleDataBuf == NULL) { - printf("Failed to calloc %d Bytes, reason:%s\n", superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, strerror(errno)); - return NULL; - } - - fp = fopen(superTblInfo->sampleFile, "r"); - if (fp == NULL) { - printf("Failed to open sample file: %s, reason:%s\n", superTblInfo->sampleFile, strerror(errno)); - tmfree(sampleDataBuf); - return NULL; - } - int ret = readSampleFromCsvFileToMem(fp, superTblInfo, sampleDataBuf); - if (0 != ret) { - tmfree(sampleDataBuf); - tmfclose(fp); - return NULL; - } - } - - if (superTblInfo->numberOfTblInOneSql > 0) { - syncWriteForNumberOfTblInOneSql(winfo, fp, sampleDataBuf); - tmfree(sampleDataBuf); - tmfclose(fp); - return NULL; - } - - //printf("========threadID[%d], table rang: %d - %d \n", winfo->threadID, winfo->start_table_id, winfo->end_table_id); - - char* buffer = calloc(superTblInfo->maxSqlLen, 1); - - int nrecords_per_request = 0; - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { - nrecords_per_request = (superTblInfo->maxSqlLen - 1280 - superTblInfo->lenOfTagOfOneRow) / superTblInfo->lenOfOneRow; - } else { - nrecords_per_request = (superTblInfo->maxSqlLen - 1280) / superTblInfo->lenOfOneRow; - } - - int nrecords_no_last_req = nrecords_per_request; - int nrecords_last_req = 0; - int loop_cnt = 0; - if (0 != superTblInfo->insertRate) { - if (nrecords_no_last_req >= superTblInfo->insertRate) { - nrecords_no_last_req = superTblInfo->insertRate; - } else { - nrecords_last_req = superTblInfo->insertRate % nrecords_per_request; - loop_cnt = (superTblInfo->insertRate / nrecords_per_request) + (superTblInfo->insertRate % nrecords_per_request ? 1 : 0) ; - } - } - - if (nrecords_no_last_req <= 0) { - nrecords_no_last_req = 1; - } - - if (nrecords_no_last_req >= INT16_MAX) { - nrecords_no_last_req = INT16_MAX - 1; - } - - if (nrecords_last_req >= INT16_MAX) { - nrecords_last_req = INT16_MAX - 1; - } - - int nrecords_cur_req = nrecords_no_last_req; - int loop_cnt_orig = loop_cnt; - - //printf("========nrecords_per_request:%d, nrecords_no_last_req:%d, nrecords_last_req:%d, loop_cnt:%d\n", nrecords_per_request, nrecords_no_last_req, nrecords_last_req, loop_cnt); - - int64_t time_counter = winfo->start_time; - - int64_t st = 0; - int64_t et = 0; - for (int i = 0; i < superTblInfo->insertRows;) { - if (superTblInfo->insertRate && (et - st) < 1000) { - taosMsleep(1000 - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); - } - - if (superTblInfo->insertRate) { - st = taosGetTimestampMs(); - } - - for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { - int inserted = i; - int64_t tmp_time = time_counter; - - int sampleUsePos = samplePos; - int k = 0; - while (1) - { - int len = 0; - memset(buffer, 0, superTblInfo->maxSqlLen); - char *pstr = buffer; - - if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { - char* tagsValBuf = NULL; - if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo); - } else { - tagsValBuf = getTagValueFromTagSample(superTblInfo, tID % superTblInfo->tagSampleCount); - } - if (NULL == tagsValBuf) { - goto free_and_statistics_2; - } - - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d using %s.%s tags %s values", winfo->db_name, superTblInfo->childTblPrefix, tID, winfo->db_name, superTblInfo->sTblName, tagsValBuf); - tmfree(tagsValBuf); - } else if (TBL_ALREADY_EXISTS == superTblInfo->childTblExists) { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s values", winfo->db_name, superTblInfo->childTblName + tID * TSDB_TABLE_NAME_LEN); - } else { - len += snprintf(pstr + len, superTblInfo->maxSqlLen - len, "insert into %s.%s%d values", winfo->db_name, superTblInfo->childTblPrefix, tID); - } - - for (k = 0; k < nrecords_cur_req;) { - int retLen = 0; - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { - retLen = getRowDataFromSample(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo, &sampleUsePos, fp, sampleDataBuf); - if (retLen < 0) { - goto free_and_statistics_2; - } - } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", 8)) { - int rand_num = rand_tinyint() % 100; - if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { - int64_t d = tmp_time - rand() % superTblInfo->disorderRange; - retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, d, superTblInfo); - //printf("disorder rows, rand_num:%d, last ts:%"PRId64" current ts:%"PRId64"\n", rand_num, tmp_time, d); - } else { - retLen = generateRowData(pstr + len, superTblInfo->maxSqlLen - len, tmp_time += superTblInfo->timeStampStep, superTblInfo); - } - if (retLen < 0) { - goto free_and_statistics_2; - } - } - len += retLen; - inserted++; - k++; - totalRowsInserted++; - - if (inserted >= superTblInfo->insertRows || (superTblInfo->maxSqlLen - len) < (superTblInfo->lenOfOneRow + 128)) break; - } - - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { - //printf("===== sql: %s \n\n", buffer); - //int64_t t1 = taosGetTimestampMs(); - int affectedRows = queryDbExec(winfo->taos, buffer, INSERT_TYPE); - if (0 > affectedRows){ - goto free_and_statistics_2; - } - totalAffectedRows += affectedRows; - - int64_t currentPrintTime = taosGetTimestampMs(); - if (currentPrintTime - lastPrintTime > 30*1000) { - printf("thread[%d] has currently inserted rows: %"PRId64 ", affected rows: %"PRId64 "\n", winfo->threadID, totalRowsInserted, totalAffectedRows); - lastPrintTime = currentPrintTime; - } - //int64_t t2 = taosGetTimestampMs(); - //printf("taosc insert sql return, Spent %.4f seconds \n", (double)(t2 - t1)/1000.0); - } else { - #ifdef TD_LOWA_CURL - //int64_t t1 = taosGetTimestampMs(); - int retCode = curlProceSql(g_Dbs.host, g_Dbs.port, buffer, winfo->curl_handle); - //int64_t t2 = taosGetTimestampMs(); - //printf("http insert sql return, Spent %ld ms \n", t2 - t1); - - if (0 != retCode) { - printf("========curl return fail, threadID[%d]\n", winfo->threadID); - goto free_and_statistics_2; - } - #else - printf("========no use http mode for no curl lib!\n"); - goto free_and_statistics_2; - #endif - } - - //printf("========tID:%d, k:%d, loop_cnt:%d\n", tID, k, loop_cnt); - - if (loop_cnt) { - loop_cnt--; - if ((1 == loop_cnt) && (0 != nrecords_last_req)) { - nrecords_cur_req = nrecords_last_req; - } else if (0 == loop_cnt){ - nrecords_cur_req = nrecords_no_last_req; - loop_cnt = loop_cnt_orig; - break; - } - } else { - break; - } - } - - if (tID == winfo->end_table_id) { - if (0 == strncasecmp(superTblInfo->dataSource, "sample", 6)) { - samplePos = sampleUsePos; - } - i = inserted; - time_counter = tmp_time; - } - } - - if (superTblInfo->insertRate) { - et = taosGetTimestampMs(); - } - //printf("========loop %d childTables duration:%"PRId64 "========inserted rows:%d\n", winfo->end_table_id - winfo->start_table_id, et - st, i); - } - - free_and_statistics_2: - tmfree(buffer); - tmfree(sampleDataBuf); - tmfclose(fp); - - winfo->totalRowsInserted = totalRowsInserted; - winfo->totalAffectedRows = totalAffectedRows; - - printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", winfo->threadID, totalRowsInserted, totalAffectedRows); - return NULL; -} - -void callBack(void *param, TAOS_RES *res, int code) { - threadInfo* winfo = (threadInfo*)param; - - if (winfo->superTblInfo->insertRate) { - winfo->et = taosGetTimestampMs(); - if (winfo->et - winfo->st < 1000) { - taosMsleep(1000 - (winfo->et - winfo->st)); // ms - } - } - - char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); - char *data = calloc(1, MAX_DATA_SIZE); - char *pstr = buffer; - pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_id); - if (winfo->counter >= winfo->superTblInfo->insertRows) { - winfo->start_table_id++; - winfo->counter = 0; - } - if (winfo->start_table_id > winfo->end_table_id) { - tsem_post(&winfo->lock_sem); - free(buffer); - free(data); - taos_free_result(res); - return; - } - - for (int i = 0; i < winfo->nrecords_per_request; i++) { - int rand_num = rand() % 100; - if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) - { - int64_t d = winfo->lastTs - rand() % 1000000 + rand_num; - //generateData(data, datatype, ncols_per_record, d, len_of_binary); - (void)generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo); - } else { - //generateData(data, datatype, ncols_per_record, tmp_time += 1000, len_of_binary); - (void)generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo); - } - pstr += sprintf(pstr, "%s", data); - winfo->counter++; - - if (winfo->counter >= winfo->superTblInfo->insertRows) { - break; - } - } - - if (winfo->superTblInfo->insertRate) { - winfo->st = taosGetTimestampMs(); - } - taos_query_a(winfo->taos, buffer, callBack, winfo); - free(buffer); - free(data); - - taos_free_result(res); -} - -void *asyncWrite(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; - - winfo->nrecords_per_request = 0; - //if (AUTO_CREATE_SUBTBL == winfo->superTblInfo->autoCreateTable) { - winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280 - winfo->superTblInfo->lenOfTagOfOneRow) / winfo->superTblInfo->lenOfOneRow; - //} else { - // winfo->nrecords_per_request = (winfo->superTblInfo->maxSqlLen - 1280) / winfo->superTblInfo->lenOfOneRow; - //} - - if (0 != winfo->superTblInfo->insertRate) { - if (winfo->nrecords_per_request >= winfo->superTblInfo->insertRate) { - winfo->nrecords_per_request = winfo->superTblInfo->insertRate; - } - } - - if (winfo->nrecords_per_request <= 0) { - winfo->nrecords_per_request = 1; - } - - if (winfo->nrecords_per_request >= INT16_MAX) { - winfo->nrecords_per_request = INT16_MAX - 1; - } - - if (winfo->nrecords_per_request >= INT16_MAX) { - winfo->nrecords_per_request = INT16_MAX - 1; - } - - winfo->st = 0; - winfo->et = 0; - winfo->lastTs = winfo->start_time; - - if (winfo->superTblInfo->insertRate) { - winfo->st = taosGetTimestampMs(); - } - taos_query_a(winfo->taos, "show databases", callBack, winfo); - - tsem_wait(&(winfo->lock_sem)); - - return NULL; -} - -void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSuperTable* superTblInfo) { - pthread_t *pids = malloc(threads * sizeof(pthread_t)); - threadInfo *infos = malloc(threads * sizeof(threadInfo)); - memset(pids, 0, threads * sizeof(pthread_t)); - memset(infos, 0, threads * sizeof(threadInfo)); - int ntables = superTblInfo->childTblCount; - - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int b = 0; - if (threads != 0) { - b = ntables % threads; - } - - TAOS* taos; - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { - taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port); - if (NULL == taos) { - printf("connect to server fail, reason: %s\n", taos_errstr(NULL)); - exit(-1); - } - } - - int32_t timePrec = TSDB_TIME_PRECISION_MILLI; - if (0 != precision[0]) { - if (0 == strncasecmp(precision, "ms", 2)) { - timePrec = TSDB_TIME_PRECISION_MILLI; - } else if (0 == strncasecmp(precision, "us", 2)) { - timePrec = TSDB_TIME_PRECISION_MICRO; - } else { - printf("No support precision: %s\n", precision); - exit(-1); - } - } - - int64_t start_time; - if (0 == strncasecmp(superTblInfo->startTimestamp, "now", 3)) { - start_time = taosGetTimestamp(timePrec); - } else { - (void)taosParseTime(superTblInfo->startTimestamp, &start_time, strlen(superTblInfo->startTimestamp), timePrec, 0); - } - - double start = getCurrentTime(); - - int last = 0; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - tstrncpy(t_info->db_name, db_name, MAX_DB_NAME_SIZE); - t_info->superTblInfo = superTblInfo; - - t_info->start_time = start_time; - - if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) { - t_info->taos = taos; - } else { - t_info->taos = NULL; - #ifdef TD_LOWA_CURL - t_info->curl_handle = curl_easy_init(); - #endif - } - - if (0 == superTblInfo->multiThreadWriteOneTbl) { - t_info->start_table_id = last; - t_info->end_table_id = i < b ? last + a : last + a - 1; - last = t_info->end_table_id + 1; - } else { - t_info->start_table_id = 0; - t_info->end_table_id = superTblInfo->childTblCount - 1; - t_info->start_time = t_info->start_time + rand_int() % 10000 - rand_tinyint(); - } - - tsem_init(&(t_info->lock_sem), 0, 0); - - if (SYNC == g_Dbs.queryMode) { - pthread_create(pids + i, NULL, syncWrite, t_info); - } else { - pthread_create(pids + i, NULL, asyncWrite, t_info); - } - } - - for (int i = 0; i < threads; i++) { - pthread_join(pids[i], NULL); - } - - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infos + i; - - tsem_destroy(&(t_info->lock_sem)); - - superTblInfo->totalAffectedRows += t_info->totalAffectedRows; - superTblInfo->totalRowsInserted += t_info->totalRowsInserted; - #ifdef TD_LOWA_CURL - if (t_info->curl_handle) { - curl_easy_cleanup(t_info->curl_handle); - } - #endif - } - - double end = getCurrentTime(); - - taos_close(taos); - - free(pids); - free(infos); - - printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s\n\n", - end - start, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName); - fprintf(g_fpOfInsertResult, "Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s) into %s.%s\n\n", - end - start, superTblInfo->totalRowsInserted, superTblInfo->totalAffectedRows, threads, db_name, superTblInfo->sTblName); -} - - -void *readTable(void *sarg) { -#if 1 - threadInfo *rinfo = (threadInfo *)sarg; - TAOS *taos = rinfo->taos; - char command[BUFFER_SIZE] = "\0"; - int64_t sTime = rinfo->start_time; - char *tb_prefix = rinfo->tb_prefix; - FILE *fp = fopen(rinfo->fp, "a"); - if (NULL == fp) { - printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); - return NULL; - } - - int num_of_DPT = rinfo->superTblInfo->insertRows; // nrecords_per_table; - int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; - int totalData = num_of_DPT * num_of_tables; - bool do_aggreFunc = g_Dbs.do_aggreFunc; - - int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; - if (!do_aggreFunc) { - printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); - } - printf("%d records:\n", totalData); - fprintf(fp, "| QFunctions | QRecords | QSpeed(R/s) | QLatency(ms) |\n"); - - for (int j = 0; j < n; j++) { - double totalT = 0; - int count = 0; - for (int i = 0; i < num_of_tables; i++) { - sprintf(command, "select %s from %s%d where ts>= %" PRId64, aggreFunc[j], tb_prefix, i, sTime); - - double t = getCurrentTime(); - TAOS_RES *pSql = taos_query(taos, command); - int32_t code = taos_errno(pSql); - - if (code != 0) { - fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); - taos_free_result(pSql); - taos_close(taos); - return NULL; - } - - while (taos_fetch_row(pSql) != NULL) { - count++; - } - - t = getCurrentTime() - t; - totalT += t; - - taos_free_result(pSql); - } - - fprintf(fp, "|%10s | %10d | %12.2f | %10.2f |\n", - aggreFunc[j][0] == '*' ? " * " : aggreFunc[j], totalData, - (double)(num_of_tables * num_of_DPT) / totalT, totalT * 1000); - printf("select %10s took %.6f second(s)\n", aggreFunc[j], totalT); - } - fprintf(fp, "\n"); - fclose(fp); -#endif - return NULL; -} - -void *readMetric(void *sarg) { -#if 1 - threadInfo *rinfo = (threadInfo *)sarg; - TAOS *taos = rinfo->taos; - char command[BUFFER_SIZE] = "\0"; - FILE *fp = fopen(rinfo->fp, "a"); - if (NULL == fp) { - printf("fopen %s fail, reason:%s.\n", rinfo->fp, strerror(errno)); - return NULL; - } - - int num_of_DPT = rinfo->superTblInfo->insertRows; - int num_of_tables = rinfo->end_table_id - rinfo->start_table_id + 1; - int totalData = num_of_DPT * num_of_tables; - bool do_aggreFunc = g_Dbs.do_aggreFunc; - - int n = do_aggreFunc ? (sizeof(aggreFunc) / sizeof(aggreFunc[0])) : 2; - if (!do_aggreFunc) { - printf("\nThe first field is either Binary or Bool. Aggregation functions are not supported.\n"); - } - printf("%d records:\n", totalData); - fprintf(fp, "Querying On %d records:\n", totalData); - - for (int j = 0; j < n; j++) { - char condition[BUFFER_SIZE - 30] = "\0"; - char tempS[64] = "\0"; - - int m = 10 < num_of_tables ? 10 : num_of_tables; - - for (int i = 1; i <= m; i++) { - if (i == 1) { - sprintf(tempS, "t1 = %d", i); - } else { - sprintf(tempS, " or t1 = %d ", i); - } - strcat(condition, tempS); - - sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); - - printf("Where condition: %s\n", condition); - fprintf(fp, "%s\n", command); - - double t = getCurrentTime(); - - TAOS_RES *pSql = taos_query(taos, command); - int32_t code = taos_errno(pSql); - - if (code != 0) { - fprintf(stderr, "Failed to query:%s\n", taos_errstr(pSql)); - taos_free_result(pSql); - taos_close(taos); - return NULL; - } - int count = 0; - while (taos_fetch_row(pSql) != NULL) { - count++; - } - t = getCurrentTime() - t; - - fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", num_of_tables * num_of_DPT / t, t * 1000); - printf("select %10s took %.6f second(s)\n\n", aggreFunc[j], t); - - taos_free_result(pSql); - } - fprintf(fp, "\n"); - } - fclose(fp); -#endif - return NULL; -} - - -int insertTestProcess() { - - g_fpOfInsertResult = fopen(g_Dbs.resultFile, "a"); - if (NULL == g_fpOfInsertResult) { - fprintf(stderr, "Failed to open %s for save result\n", g_Dbs.resultFile); - return 1; - }; - - printfInsertMeta(); - printfInsertMetaToFile(g_fpOfInsertResult); - - printf("Press enter key to continue\n\n"); - (void)getchar(); - - init_rand_data(); - - // create database and super tables - (void)createDatabases(); - - // pretreatement - prePareSampleData(); - - double start; - double end; - - // create child tables - start = getCurrentTime(); - createChildTables(); - end = getCurrentTime(); - if (g_totalChildTables > 0) { - printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); - fprintf(g_fpOfInsertResult, "Spent %.4f seconds to create %d tables with %d thread(s)\n\n", end - start, g_totalChildTables, g_Dbs.threadCount); - } - - usleep(1000*1000); - - // create sub threads for inserting data - //start = getCurrentTime(); - for (int i = 0; i < g_Dbs.dbCount; i++) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; - startMultiThreadInsertData(g_Dbs.threadCount, g_Dbs.db[i].dbName, g_Dbs.db[i].dbCfg.precision, superTblInfo); - } - } - //end = getCurrentTime(); - - //int64_t totalRowsInserted = 0; - //int64_t totalAffectedRows = 0; - //for (int i = 0; i < g_Dbs.dbCount; i++) { - // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - // totalRowsInserted += g_Dbs.db[i].superTbls[j].totalRowsInserted; - // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; - //} - //printf("Spent %.4f seconds to insert rows: %"PRId64", affected rows: %"PRId64" with %d thread(s)\n\n", end - start, totalRowsInserted, totalAffectedRows, g_Dbs.threadCount); - if (NULL == g_args.metaFile && false == g_Dbs.insert_only) { - // query data - pthread_t read_id; - threadInfo *rInfo = malloc(sizeof(threadInfo)); - rInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 - rInfo->start_table_id = 0; - rInfo->end_table_id = g_Dbs.db[0].superTbls[0].childTblCount - 1; - //rInfo->do_aggreFunc = g_Dbs.do_aggreFunc; - //rInfo->nrecords_per_table = g_Dbs.db[0].superTbls[0].insertRows; - rInfo->superTblInfo = &g_Dbs.db[0].superTbls[0]; - rInfo->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); - strcpy(rInfo->tb_prefix, g_Dbs.db[0].superTbls[0].childTblPrefix); - strcpy(rInfo->fp, g_Dbs.resultFile); - - if (!g_Dbs.use_metric) { - pthread_create(&read_id, NULL, readTable, rInfo); - } else { - pthread_create(&read_id, NULL, readMetric, rInfo); - } - pthread_join(read_id, NULL); - taos_close(rInfo->taos); - } - - postFreeResource(); - - return 0; -} - -void *superQueryProcess(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; - - //char sqlStr[MAX_TB_NAME_SIZE*2]; - //sprintf(sqlStr, "use %s", g_queryInfo.dbName); - //queryDB(winfo->taos, sqlStr); - - int64_t st = 0; - int64_t et = 0; - while (1) { - if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { - taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); - } - - st = taosGetTimestampMs(); - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - int64_t t1 = taosGetTimestampUs(); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); - } - selectAndGetResult(winfo->taos, g_queryInfo.superQueryInfo.sql[i], tmpFile); - int64_t t2 = taosGetTimestampUs(); - printf("taosc select sql return, Spent %f s\n", (t2 - t1)/1000000.0); - } else { - #ifdef TD_LOWA_CURL - int64_t t1 = taosGetTimestampUs(); - int retCode = curlProceSql(g_queryInfo.host, g_queryInfo.port, g_queryInfo.superQueryInfo.sql[i], winfo->curl_handle); - int64_t t2 = taosGetTimestampUs(); - printf("http select sql return, Spent %f s \n", (t2 - t1)/1000000.0); - - if (0 != retCode) { - printf("========curl return fail, threadID[%d]\n", winfo->threadID); - return NULL; - } - #endif - } - } - et = taosGetTimestampMs(); - printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.6fs\n\n", (uint64_t)pthread_self(), (double)(et - st)/1000.0); - } - return NULL; -} - -void replaceSubTblName(char* inSql, char* outSql, int tblIndex) { - char sourceString[32] = "xxxx"; - char subTblName[MAX_TB_NAME_SIZE*3]; - sprintf(subTblName, "%s.%s", g_queryInfo.dbName, g_queryInfo.subQueryInfo.childTblName + tblIndex*TSDB_TABLE_NAME_LEN); - - //printf("inSql: %s\n", inSql); - - char* pos = strstr(inSql, sourceString); - if (0 == pos) { - return; - } - - strncpy(outSql, inSql, pos - inSql); - //printf("1: %s\n", outSql); - strcat(outSql, subTblName); - //printf("2: %s\n", outSql); - strcat(outSql, pos+strlen(sourceString)); - //printf("3: %s\n", outSql); -} - -void *subQueryProcess(void *sarg) { - char sqlstr[1024]; - threadInfo *winfo = (threadInfo *)sarg; - int64_t st = 0; - int64_t et = 0; - while (1) { - if (g_queryInfo.subQueryInfo.rate && (et - st) < g_queryInfo.subQueryInfo.rate*1000) { - taosMsleep(g_queryInfo.subQueryInfo.rate*1000 - (et - st)); // ms - //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); - } - - st = taosGetTimestampMs(); - for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) { - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - memset(sqlstr,0,sizeof(sqlstr)); - replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], sqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID); - } - selectAndGetResult(winfo->taos, sqlstr, tmpFile); - } - } - et = taosGetTimestampMs(); - printf("========thread[%"PRIu64"] complete all sqls to allocate all sub-tables once queries duration:%.4fs\n\n", (uint64_t)pthread_self(), (double)(et - st)/1000.0); - } - return NULL; -} - -int queryTestProcess() { - printfQueryMeta(); - - printf("Press enter key to continue\n\n"); - (void)getchar(); - - TAOS * taos = NULL; - taos_init(); - taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); - if (taos == NULL) { - fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - exit(-1); - } - - if (0 != g_queryInfo.subQueryInfo.sqlCount) { - (void)getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); - } - - pthread_t *pids = NULL; - threadInfo *infos = NULL; - //==== create sub threads for query from super table - if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { - - pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - printf("malloc failed for create threads\n"); - taos_close(taos); - exit(-1); - } - - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", 5)) { - t_info->taos = taos; - - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - (void)queryDbExec(t_info->taos, sqlStr, NO_INSERT_TYPE); - } else { - t_info->taos = NULL; - #ifdef TD_LOWA_CURL - t_info->curl_handle = curl_easy_init(); - #endif - } - - pthread_create(pids + i, NULL, superQueryProcess, t_info); - } - }else { - g_queryInfo.superQueryInfo.concurrent = 0; - } - - pthread_t *pidsOfSub = NULL; - threadInfo *infosOfSub = NULL; - //==== create sub threads for query from sub table - if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); - if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - printf("malloc failed for create threads\n"); - taos_close(taos); - exit(-1); - } - - int ntables = g_queryInfo.subQueryInfo.childTblCount; - int threads = g_queryInfo.subQueryInfo.threadCnt; - - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int b = 0; - if (threads != 0) { - b = ntables % threads; - } - - int last = 0; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; - - t_info->start_table_id = last; - t_info->end_table_id = i < b ? last + a : last + a - 1; - t_info->taos = taos; - pthread_create(pidsOfSub + i, NULL, subQueryProcess, t_info); - } - - g_queryInfo.subQueryInfo.threadCnt = threads; - }else { - g_queryInfo.subQueryInfo.threadCnt = 0; - } - - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { - pthread_join(pids[i], NULL); - } - - tmfree((char*)pids); - tmfree((char*)infos); - - for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { - pthread_join(pidsOfSub[i], NULL); - } - - tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); - - taos_close(taos); - return 0; -} - -static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { - if (res == NULL || taos_errno(res) != 0) { - printf("failed to subscribe result, code:%d, reason:%s\n", code, taos_errstr(res)); - return; - } - - getResult(res, (char*)param); - taos_free_result(res); -} - -static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultFileName) { - TAOS_SUB* tsub = NULL; - - if (g_queryInfo.superQueryInfo.subscribeMode) { - tsub = taos_subscribe(taos, g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, subscribe_callback, (void*)resultFileName, g_queryInfo.superQueryInfo.subscribeInterval); - } else { - tsub = taos_subscribe(taos, g_queryInfo.superQueryInfo.subscribeRestart, topic, sql, NULL, NULL, 0); - } - - if (tsub == NULL) { - printf("failed to create subscription. topic:%s, sql:%s\n", topic, sql); - return NULL; - } - - return tsub; -} - -void *subSubscribeProcess(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; - char subSqlstr[1024]; - - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)){ - return NULL; - } - - //int64_t st = 0; - //int64_t et = 0; - do { - //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { - // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms - // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); - //} - - //st = taosGetTimestampMs(); - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); - memset(subSqlstr,0,sizeof(subSqlstr)); - replaceSubTblName(g_queryInfo.subQueryInfo.sql[i], subSqlstr, i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID); - } - g_queryInfo.subQueryInfo.tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); - if (NULL == g_queryInfo.subQueryInfo.tsub[i]) { - return NULL; - } - } - //et = taosGetTimestampMs(); - //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", pthread_self(), (double)(et - st)/1000.0); - } while (0); - - // start loop to consume result - TAOS_RES* res = NULL; - while (1) { - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - if (1 == g_queryInfo.subQueryInfo.subscribeMode) { - continue; - } - - res = taos_consume(g_queryInfo.subQueryInfo.tsub[i]); - if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.subQueryInfo.result[i], winfo->threadID); - } - getResult(res, tmpFile); - } - } - } - taos_free_result(res); - - for (int i = 0; i < g_queryInfo.subQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.subQueryInfo.tsub[i], g_queryInfo.subQueryInfo.subscribeKeepProgress); - } - return NULL; -} - -void *superSubscribeProcess(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; - - char sqlStr[MAX_TB_NAME_SIZE*2]; - sprintf(sqlStr, "use %s", g_queryInfo.dbName); - if (0 != queryDbExec(winfo->taos, sqlStr, NO_INSERT_TYPE)) { - return NULL; - } - - //int64_t st = 0; - //int64_t et = 0; - do { - //if (g_queryInfo.superQueryInfo.rate && (et - st) < g_queryInfo.superQueryInfo.rate*1000) { - // taosMsleep(g_queryInfo.superQueryInfo.rate*1000 - (et - st)); // ms - // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, winfo->start_table_id, winfo->end_table_id); - //} - - //st = taosGetTimestampMs(); - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.subQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); - } - g_queryInfo.superQueryInfo.tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.superQueryInfo.sql[i], topic, tmpFile); - if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { - return NULL; - } - } - //et = taosGetTimestampMs(); - //printf("========thread[%"PRId64"] complete all sqls to super table once queries duration:%.4fs\n", pthread_self(), (double)(et - st)/1000.0); - } while (0); - - // start loop to consume result - TAOS_RES* res = NULL; - while (1) { - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (1 == g_queryInfo.superQueryInfo.subscribeMode) { - continue; - } - - res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); - if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); - } - getResult(res, tmpFile); - } - } - } - taos_free_result(res); - - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); - } - return NULL; -} - -int subscribeTestProcess() { - printfQueryMeta(); - - printf("Press enter key to continue\n\n"); - (void)getchar(); - - TAOS * taos = NULL; - taos_init(); - taos = taos_connect(g_queryInfo.host, g_queryInfo.user, g_queryInfo.password, g_queryInfo.dbName, g_queryInfo.port); - if (taos == NULL) { - fprintf(stderr, "Failed to connect to TDengine, reason:%s\n", taos_errstr(NULL)); - exit(-1); - } - - if (0 != g_queryInfo.subQueryInfo.sqlCount) { - (void)getAllChildNameOfSuperTable(taos, g_queryInfo.dbName, g_queryInfo.subQueryInfo.sTblName, &g_queryInfo.subQueryInfo.childTblName, &g_queryInfo.subQueryInfo.childTblCount); - } - - - pthread_t *pids = NULL; - threadInfo *infos = NULL; - //==== create sub threads for query from super table - if (g_queryInfo.superQueryInfo.sqlCount > 0 && g_queryInfo.superQueryInfo.concurrent > 0) { - pids = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.superQueryInfo.concurrent * sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - printf("malloc failed for create threads\n"); - taos_close(taos); - exit(-1); - } - - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { - threadInfo *t_info = infos + i; - t_info->threadID = i; - t_info->taos = taos; - pthread_create(pids + i, NULL, superSubscribeProcess, t_info); - } - } - - //==== create sub threads for query from sub table - pthread_t *pidsOfSub = NULL; - threadInfo *infosOfSub = NULL; - if ((g_queryInfo.subQueryInfo.sqlCount > 0) && (g_queryInfo.subQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.subQueryInfo.threadCnt * sizeof(threadInfo)); - if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { - printf("malloc failed for create threads\n"); - taos_close(taos); - exit(-1); - } - - int ntables = g_queryInfo.subQueryInfo.childTblCount; - int threads = g_queryInfo.subQueryInfo.threadCnt; - - int a = ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; - } - - int b = 0; - if (threads != 0) { - b = ntables % threads; - } - - int last = 0; - for (int i = 0; i < threads; i++) { - threadInfo *t_info = infosOfSub + i; - t_info->threadID = i; - - t_info->start_table_id = last; - t_info->end_table_id = i < b ? last + a : last + a - 1; - t_info->taos = taos; - pthread_create(pidsOfSub + i, NULL, subSubscribeProcess, t_info); - } - g_queryInfo.subQueryInfo.threadCnt = threads; - } - - for (int i = 0; i < g_queryInfo.superQueryInfo.concurrent; i++) { - pthread_join(pids[i], NULL); - } - - tmfree((char*)pids); - tmfree((char*)infos); - - for (int i = 0; i < g_queryInfo.subQueryInfo.threadCnt; i++) { - pthread_join(pidsOfSub[i], NULL); - } - - tmfree((char*)pidsOfSub); - tmfree((char*)infosOfSub); - taos_close(taos); - return 0; -} - -void initOfInsertMeta() { - memset(&g_Dbs, 0, sizeof(SDbs)); - - // set default values - strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); - g_Dbs.port = 6030; - strncpy(g_Dbs.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); - strncpy(g_Dbs.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); - g_Dbs.threadCount = 2; - g_Dbs.use_metric = true; -} - -void initOfQueryMeta() { - memset(&g_queryInfo, 0, sizeof(SQueryMetaInfo)); - - // set default values - strncpy(g_queryInfo.host, "127.0.0.1", MAX_DB_NAME_SIZE); - g_queryInfo.port = 6030; - strncpy(g_queryInfo.user, TSDB_DEFAULT_USER, MAX_DB_NAME_SIZE); - strncpy(g_queryInfo.password, TSDB_DEFAULT_PASS, MAX_DB_NAME_SIZE); -} - -void setParaFromArg(){ - if (g_args.host) { - strcpy(g_Dbs.host, g_args.host); - } else { - strncpy(g_Dbs.host, "127.0.0.1", MAX_DB_NAME_SIZE); - } - - if (g_args.user) { - strcpy(g_Dbs.user, g_args.user); - } - - if (g_args.password) { - strcpy(g_Dbs.password, g_args.password); - } - - if (g_args.port) { - g_Dbs.port = g_args.port; - } - - g_Dbs.dbCount = 1; - g_Dbs.db[0].drop = 1; - - strncpy(g_Dbs.db[0].dbName, g_args.database, MAX_DB_NAME_SIZE); - g_Dbs.db[0].dbCfg.replica = g_args.replica; - strncpy(g_Dbs.db[0].dbCfg.precision, "ms", MAX_DB_NAME_SIZE); - - - strncpy(g_Dbs.resultFile, g_args.output_file, MAX_FILE_NAME_LEN); - - g_Dbs.use_metric = g_args.use_metric; - g_Dbs.insert_only = g_args.insert_only; - - g_Dbs.db[0].superTblCount = 1; - strncpy(g_Dbs.db[0].superTbls[0].sTblName, "meters", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; - g_Dbs.threadCount = g_args.num_of_threads; - g_Dbs.queryMode = g_args.mode; - - g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; - g_Dbs.db[0].superTbls[0].superTblExists = TBL_NO_EXISTS; - g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; - g_Dbs.db[0].superTbls[0].insertRate = 0; - g_Dbs.db[0].superTbls[0].disorderRange = g_args.disorderRange; - g_Dbs.db[0].superTbls[0].disorderRatio = g_args.disorderRatio; - strncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, MAX_TB_NAME_SIZE); - strncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); - strncpy(g_Dbs.db[0].superTbls[0].insertMode, "taosc", MAX_TB_NAME_SIZE); - strncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].timeStampStep = 10; - - // g_args.num_of_RPR; - g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; - g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; - - g_Dbs.do_aggreFunc = true; - - char dataString[STRING_LEN]; - char **data_type = g_args.datatype; - - memset(dataString, 0, STRING_LEN); - - if (strcasecmp(data_type[0], "BINARY") == 0 || strcasecmp(data_type[0], "BOOL") == 0 || strcasecmp(data_type[0], "NCHAR") == 0 ) { - g_Dbs.do_aggreFunc = false; - } - - g_Dbs.db[0].superTbls[0].columnCount = 0; - for (int i = 0; i < MAX_NUM_DATATYPE; i++) { - if (data_type[i] == NULL) { - break; - } - - strncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, data_type[i], MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = g_args.len_of_binary; - g_Dbs.db[0].superTbls[0].columnCount++; - } - - if (g_Dbs.db[0].superTbls[0].columnCount > g_args.num_of_CPR) { - g_Dbs.db[0].superTbls[0].columnCount = g_args.num_of_CPR; - } else { - for (int i = g_Dbs.db[0].superTbls[0].columnCount; i < g_args.num_of_CPR; i++) { - strncpy(g_Dbs.db[0].superTbls[0].columns[i].dataType, "INT", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].columns[i].dataLen = 0; - g_Dbs.db[0].superTbls[0].columnCount++; - } - } - - if (g_Dbs.use_metric) { - strncpy(g_Dbs.db[0].superTbls[0].tags[0].dataType, "INT", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].tags[0].dataLen = 0; - - strncpy(g_Dbs.db[0].superTbls[0].tags[1].dataType, "BINARY", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].tags[1].dataLen = g_args.len_of_binary; - g_Dbs.db[0].superTbls[0].tagCount = 2; - } else { - g_Dbs.db[0].superTbls[0].tagCount = 0; - } -} - -/* Function to do regular expression check */ -static int regexMatch(const char *s, const char *reg, int cflags) { - regex_t regex; - char msgbuf[100] = {0}; - - /* Compile regular expression */ - if (regcomp(®ex, reg, cflags) != 0) { - printf("Fail to compile regex\n"); - exit(-1); - } - - /* Execute regular expression */ - int reti = regexec(®ex, s, 0, NULL, 0); - if (!reti) { - regfree(®ex); - return 1; - } else if (reti == REG_NOMATCH) { - regfree(®ex); - return 0; - } else { - regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); - printf("Regex match failed: %s\n", msgbuf); - regfree(®ex); - exit(-1); - } - - return 0; -} - -static int isCommentLine(char *line) { - if (line == NULL) return 1; - - return regexMatch(line, "^\\s*#.*", REG_EXTENDED); -} - -void querySqlFile(TAOS* taos, char* sqlFile) -{ - FILE *fp = fopen(sqlFile, "r"); - if (fp == NULL) { - printf("failed to open file %s, reason:%s\n", sqlFile, strerror(errno)); - return; - } - - int read_len = 0; - char * cmd = calloc(1, MAX_SQL_SIZE); - size_t cmd_len = 0; - char * line = NULL; - size_t line_len = 0; - - double t = getCurrentTime(); - - while ((read_len = tgetline(&line, &line_len, fp)) != -1) { - if (read_len >= MAX_SQL_SIZE) continue; - line[--read_len] = '\0'; - - if (read_len == 0 || isCommentLine(line)) { // line starts with # - continue; - } - - if (line[read_len - 1] == '\\') { - line[read_len - 1] = ' '; - memcpy(cmd + cmd_len, line, read_len); - cmd_len += read_len; - continue; - } - - memcpy(cmd + cmd_len, line, read_len); - queryDbExec(taos, cmd, NO_INSERT_TYPE); - memset(cmd, 0, MAX_SQL_SIZE); - cmd_len = 0; - } - - t = getCurrentTime() - t; - printf("run %s took %.6f second(s)\n\n", sqlFile, t); - - tmfree(cmd); - tmfree(line); - tmfclose(fp); - return; -} - -int main(int argc, char *argv[]) { - parse_args(argc, argv, &g_args); - - if (g_args.metaFile) { - initOfInsertMeta(); - initOfQueryMeta(); - if (false == getInfoFromJsonFile(g_args.metaFile)) { - printf("Failed to read %s\n", g_args.metaFile); - return 1; - } - } else { - - memset(&g_Dbs, 0, sizeof(SDbs)); - g_jsonType = INSERT_MODE; - setParaFromArg(); - - if (NULL != g_args.sqlFile) { - TAOS* qtaos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, g_Dbs.db[0].dbName, g_Dbs.port); - querySqlFile(qtaos, g_args.sqlFile); - taos_close(qtaos); - return 0; - } - - (void)insertTestProcess(); - if (g_Dbs.insert_only) return 0; - - // select - - //printf("At present, there is no integration of taosdemo, please wait patiently!\n"); - return 0; - } - - if (INSERT_MODE == g_jsonType) { - if (g_Dbs.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_Dbs.cfgDir); - (void)insertTestProcess(); - } else if (QUERY_MODE == g_jsonType) { - if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); - (void)queryTestProcess(); - } else if (SUBSCRIBE_MODE == g_jsonType) { - if (g_queryInfo.cfgDir[0]) taos_options(TSDB_OPTION_CONFIGDIR, g_queryInfo.cfgDir); - (void)subscribeTestProcess(); - } else { - ; - } - - taos_cleanup(); - return 0; -} - diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index dc9ac6d4a789c4292480f522240690e13b47245b..58897b89e95743c802755c0476f3b2843a244a59 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) @@ -12,5 +12,15 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(taosdump taos_static) ELSE () TARGET_LINK_LIBRARIES(taosdump taos) - ENDIF () + ENDIF () +ENDIF () + +IF (TD_DARWIN) + # missing for macosx + # ADD_EXECUTABLE(taosdump ${SRC}) + # IF (TD_SOMODE_STATIC) + # TARGET_LINK_LIBRARIES(taosdump taos_static) + # ELSE () + # TARGET_LINK_LIBRARIES(taosdump taos) + # ENDIF () ENDIF () diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 60707f22e297e3cd253d88cc9bed962dd36dacd1..3cee5f1b1dc02e0218c97e66c4af9acb6c77c69f 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -769,6 +769,7 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu } sprintf(tmpBuf, ".select-tbname.tmp"); (void)remove(tmpBuf); + free(tblBuf); close(fd); return -1; } @@ -793,6 +794,7 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu *totalNumOfThread = numOfThread; + free(tblBuf); return 0; } @@ -1522,6 +1524,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao } sprintf(tmpBuf, ".show-tables.tmp"); (void)remove(tmpBuf); + free(tblBuf); close(fd); return -1; } @@ -1553,6 +1556,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao (void)remove(tmpBuf); } + free(tblBuf); return 0; } @@ -2107,13 +2111,15 @@ static void taosGetDirectoryFileList(char *inputDir) if (fileStat.st_mode & S_IFDIR) { taosCheckTablesSQLFile(inputDir); tsSqlFileNum = taosGetFilesNum(inputDir, "sql"); - int totalSQLFileNum = tsSqlFileNum; + int tsSqlFileNumOfTbls = tsSqlFileNum; if (tsDbSqlFile[0] != 0) { - tsSqlFileNum--; + tsSqlFileNumOfTbls--; } taosMallocSQLFiles(); - taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNum); - fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir); + if (0 != tsSqlFileNumOfTbls) { + taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNumOfTbls); + } + fprintf(stdout, "\nstart to dispose %d files in %s\n", tsSqlFileNum, inputDir); } else { fprintf(stderr, "ERROR: %s is not a directory\n", inputDir); @@ -2266,7 +2272,10 @@ int taosDumpIn(struct arguments *arguments) { taosGetDirectoryFileList(arguments->inpath); + int32_t tsSqlFileNumOfTbls = tsSqlFileNum; if (tsDbSqlFile[0] != 0) { + tsSqlFileNumOfTbls--; + fp = taosOpenDumpInFile(tsDbSqlFile); if (NULL == fp) { fprintf(stderr, "failed to open input file %s\n", tsDbSqlFile); @@ -2279,7 +2288,9 @@ int taosDumpIn(struct arguments *arguments) { taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile); } - taosStartDumpInWorkThreads(taos, arguments); + if (0 != tsSqlFileNumOfTbls) { + taosStartDumpInWorkThreads(taos, arguments); + } taos_close(taos); taosFreeSQLFiles(); diff --git a/src/mnode/CMakeLists.txt b/src/mnode/CMakeLists.txt index 41230986948c7ddbd746e2bd2c6e5fcece66c041..2df4708c239515febafc7a4f3ab3f63bd9e434e8 100644 --- a/src/mnode/CMakeLists.txt +++ b/src/mnode/CMakeLists.txt @@ -1,12 +1,10 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_LINUX) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) +INCLUDE_DIRECTORIES(inc) +AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(mnode ${SRC}) -ENDIF () \ No newline at end of file +ADD_LIBRARY(mnode ${SRC}) diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index 9354b923d74de055d3905ebe6581d3f721630b08..d03ba8d717a13cc943f2494dc0dca0d59d108b6f 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -32,7 +32,7 @@ int32_t mnodeInitDbs(); void mnodeCleanupDbs(); int64_t mnodeGetDbNum(); SDbObj *mnodeGetDb(char *db); -SDbObj *mnodeGetDbByTableId(char *db); +SDbObj *mnodeGetDbByTableName(char *db); void * mnodeGetNextDb(void *pIter, SDbObj **pDb); void mnodeCancelGetNextDb(void *pIter); void mnodeIncDbRef(SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 5eeac97209634206d42e1c4aebec129e67ee38e5..ed1de1b87a475cf5a2264abb4b8787c0841d1b63 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -175,7 +175,9 @@ typedef struct { int8_t quorum; int8_t update; int8_t cacheLastRow; - int8_t reserved[10]; + int8_t dbType; + int16_t partitions; + int8_t reserved[7]; } SDbCfg; typedef struct SDbObj { diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index 56d7455ad24f76a0b289e8ad8882db1867d0bed9..fa1995254e9598614cebfef57221b3ff4238b6f1 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -67,6 +67,7 @@ void mnodeCleanupDnodes(); int32_t mnodeGetDnodesNum(); int32_t mnodeGetOnlinDnodesCpuCoreNum(); int32_t mnodeGetOnlineDnodesNum(); +void mnodeGetOnlineAndTotalDnodesNum(int32_t *onlineNum, int32_t *totalNum); void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode); void mnodeCancelGetNextDnode(void *pIter); void mnodeIncDnodeRef(SDnodeObj *pDnode); diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 2067ad04ccf963701bffcf134c8ae478547f1abb..7b798c23f8b3414d40e504990bb75812d829102a 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -44,11 +44,12 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle); void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle); int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid); -void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable); +int32_t mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable, bool needCheck); void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable); void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle); void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); +void mnodeSendSyncVgroupMsg(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup); SRpcEpSet mnodeGetEpSetFromIp(char *ep); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 6fba05674fadbe967a673bbc0cc6e02ad7f2b41a..afe474df6b7d8d4bd1945523a95d839bcd29e247 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -81,7 +81,7 @@ static int32_t mnodeAcctActionDecode(SSdbRow *pRow) { } static int32_t mnodeAcctActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsAcctSdb); + int64_t numOfRows = sdbGetNumOfRows(tsAcctSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create root acct"); int32_t code = mnodeCreateRootAcct(); @@ -97,14 +97,14 @@ static int32_t mnodeAcctActionRestored() { int32_t mnodeInitAccts() { SAcctObj tObj; - tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsAcctUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_ACCOUNT, .name = "accounts", .hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE, .maxRowSize = tsAcctUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeAcctActionInsert, .fpDelete = mnodeAcctActionDelete, @@ -206,7 +206,7 @@ void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) { } static int32_t mnodeCreateRootAcct() { - int32_t numOfAccts = sdbGetNumOfRows(tsAcctSdb); + int64_t numOfAccts = sdbGetNumOfRows(tsAcctSdb); if (numOfAccts != 0) return TSDB_CODE_SUCCESS; SAcctObj *pAcct = malloc(sizeof(SAcctObj)); diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index a35e30481077d0e16888d0ee036312cd068ea9b4..169d2ebd9dae1bcc34ea24dd3770951f9cbb442b 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -68,7 +68,7 @@ static int32_t mnodeClusterActionDecode(SSdbRow *pRow) { } static int32_t mnodeClusterActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsClusterSdb); + int64_t numOfRows = sdbGetNumOfRows(tsClusterSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create cluster"); int32_t code = mnodeCreateCluster(); @@ -84,14 +84,14 @@ static int32_t mnodeClusterActionRestored() { int32_t mnodeInitCluster() { SClusterObj tObj; - tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsClusterUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_CLUSTER, .name = "cluster", .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE, .maxRowSize = tsClusterUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeClusterActionInsert, .fpDelete = mnodeClusterActionDelete, @@ -139,7 +139,7 @@ void mnodeDecClusterRef(SClusterObj *pCluster) { } static int32_t mnodeCreateCluster() { - int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); + int64_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); if (numOfClusters != 0) return TSDB_CODE_SUCCESS; SClusterObj *pCluster = malloc(sizeof(SClusterObj)); @@ -195,7 +195,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void * cols++; pMeta->numOfColumns = htons(cols); - strcpy(pMeta->tableId, "show cluster"); + strcpy(pMeta->tableFname, "show cluster"); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -226,7 +226,7 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *) pWrite = pCluster->createdTime; + *(int64_t *) pWrite = pCluster->createdTime; cols++; mnodeDecClusterRef(pCluster); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index fbcad151bc5f8f7f9168e12db02059b0c86d0f2c..909ca7cac6ec04af7d6c624a8d7fea06e7cad702 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -22,6 +22,7 @@ #include "tname.h" #include "tbn.h" #include "tdataformat.h" +#include "tp.h" #include "mnode.h" #include "mnodeDef.h" #include "mnodeInt.h" @@ -38,8 +39,8 @@ #include "mnodeVgroup.h" #define VG_LIST_SIZE 8 -int64_t tsDbRid = -1; -static void * tsDbSdb = NULL; +int64_t tsDbRid = -1; +void * tsDbSdb = NULL; static int32_t tsDbUpdateSize; static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg *pMsg); @@ -48,8 +49,15 @@ static int32_t mnodeSetDbDropping(SDbObj *pDb); static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg); -static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg); +static int32_t mnodeProcessSyncDbMsg(SMnodeMsg *pMsg); +int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg); + +#ifndef _TOPIC +int32_t tpInit() { return 0; } +void tpCleanUp() {} +void tpUpdateTs(int32_t vgId, int64_t *seq, void *pMsg) {} +#endif static void mnodeDestroyDb(SDbObj *pDb) { pthread_mutex_destroy(&pDb->mutex); @@ -143,14 +151,14 @@ static int32_t mnodeDbActionRestored() { int32_t mnodeInitDbs() { SDbObj tObj; - tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsDbUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_DB, .name = "dbs", .hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE, .maxRowSize = tsDbUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeDbActionInsert, .fpDelete = mnodeDbActionDelete, @@ -171,12 +179,13 @@ int32_t mnodeInitDbs() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_CREATE_DB, mnodeProcessCreateDbMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_ALTER_DB, mnodeProcessAlterDbMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mnodeProcessDropDbMsg); + mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_SYNC_DB, mnodeProcessSyncDbMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DB, mnodeGetDbMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mnodeRetrieveDbs); mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb); mDebug("table:dbs table is created"); - return 0; + return tpInit(); } void *mnodeGetNextDb(void *pIter, SDbObj **pDb) { @@ -192,25 +201,20 @@ SDbObj *mnodeGetDb(char *db) { } void mnodeIncDbRef(SDbObj *pDb) { - return sdbIncRef(tsDbSdb, pDb); + sdbIncRef(tsDbSdb, pDb); } void mnodeDecDbRef(SDbObj *pDb) { - return sdbDecRef(tsDbSdb, pDb); + sdbDecRef(tsDbSdb, pDb); } -SDbObj *mnodeGetDbByTableId(char *tableId) { - char db[TSDB_TABLE_FNAME_LEN], *pos; - - // tableId format should be : acct.db.table - pos = strstr(tableId, TS_PATH_DELIMITER); - assert(NULL != pos); +SDbObj *mnodeGetDbByTableName(char *tableName) { + SName name = {0}; + tNameFromString(&name, tableName, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE); - pos = strstr(pos + 1, TS_PATH_DELIMITER); - assert(NULL != pos); - - memset(db, 0, sizeof(db)); - strncpy(db, tableId, pos - tableId); + // validate the tableName? + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); return mnodeGetDb(db); } @@ -337,6 +341,17 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { return TSDB_CODE_MND_INVALID_DB_OPTION; } + if (pCfg->dbType < 0 || pCfg->dbType > 1) { + mError("invalid db option dbType:%d valid range: [%d, %d]", pCfg->dbType, 0, 1); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + + if (pCfg->partitions < TSDB_MIN_DB_PARTITON_OPTION || pCfg->partitions > TSDB_MAX_DB_PARTITON_OPTION) { + mError("invalid db option partitions:%d valid range: [%d, %d]", pCfg->partitions, TSDB_MIN_DB_PARTITON_OPTION, + TSDB_MAX_DB_PARTITON_OPTION); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + return TSDB_CODE_SUCCESS; } @@ -359,6 +374,8 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; if (pCfg->update < 0) pCfg->update = tsUpdate; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = tsCacheLastRow; + if (pCfg->dbType < 0) pCfg->dbType = 0; + if (pCfg->partitions < 0) pCfg->partitions = tsPartitons; } static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { @@ -413,7 +430,9 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * .replications = pCreate->replications, .quorum = pCreate->quorum, .update = pCreate->update, - .cacheLastRow = pCreate->cacheLastRow + .cacheLastRow = pCreate->cacheLastRow, + .dbType = pCreate->dbType, + .partitions = pCreate->partitions }; mnodeSetDefaultDbCfg(&pDb->cfg); @@ -506,6 +525,7 @@ void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) { } void mnodeCleanupDbs() { + tpCleanUp(); sdbCloseTable(tsDbRid); tsDbSdb = NULL; } @@ -573,7 +593,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "keep1,keep2,keep(D)"); + strcpy(pSchema[cols].name, "keep0,keep1,keep(D)"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -665,7 +685,7 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn return 0; } -static char *mnodeGetDbStr(char *src) { +char *mnodeGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); if (pos != NULL) ++pos; @@ -684,7 +704,7 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void pShow->pIter = mnodeGetNextDb(pShow->pIter, &pDb); if (pDb == NULL) break; - if (pDb->pAcct != pUser->pAcct) { + if (pDb->pAcct != pUser->pAcct || pDb->status != TSDB_DB_STATUS_READY) { mnodeDecDbRef(pDb); continue; } @@ -857,6 +877,7 @@ static int32_t mnodeProcessCreateDbMsg(SMnodeMsg *pMsg) { pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); pCreate->commitTime = htonl(pCreate->commitTime); pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); + pCreate->partitions = htons(pCreate->partitions); pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); @@ -892,6 +913,8 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { int8_t precision = pAlter->precision; int8_t update = pAlter->update; int8_t cacheLastRow = pAlter->cacheLastRow; + int8_t dbType = pAlter->dbType; + int16_t partitions = htons(pAlter->partitions); terrno = TSDB_CODE_SUCCESS; @@ -1009,6 +1032,16 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { newCfg.cacheLastRow = cacheLastRow; } + if (dbType >= 0 && dbType != pDb->cfg.dbType) { + mDebug("db:%s, dbType:%d change to %d", pDb->name, pDb->cfg.dbType, dbType); + newCfg.dbType = dbType; + } + + if (partitions >= 0 && partitions != pDb->cfg.partitions) { + mDebug("db:%s, partitions:%d change to %d", pDb->name, pDb->cfg.partitions, partitions); + newCfg.partitions = partitions; + } + return newCfg; } @@ -1036,6 +1069,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { } static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { + mDebug("db:%s, type:%d do alter operation", pDb->name, pDb->cfg.dbType); + SDbCfg newCfg = mnodeGetAlterDbOption(pDb, pAlter); if (terrno != TSDB_CODE_SUCCESS) { return terrno; @@ -1066,9 +1101,9 @@ static int32_t mnodeAlterDb(SDbObj *pDb, SAlterDbMsg *pAlter, void *pMsg) { return code; } -static int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) { +int32_t mnodeProcessAlterDbMsg(SMnodeMsg *pMsg) { SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont; - mDebug("db:%s, alter db msg is received from thandle:%p", pAlter->db, pMsg->rpcMsg.handle); + mDebug("db:%s, alter db msg is received from thandle:%p, dbType:%d", pAlter->db, pMsg->rpcMsg.handle, pAlter->dbType); if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pAlter->db); if (pMsg->pDb == NULL) { @@ -1151,6 +1186,46 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { return mnodeDropDb(pMsg); } +static int32_t mnodeSyncDb(SDbObj *pDb, SMnodeMsg *pMsg) { + void *pIter = NULL; + SVgObj *pVgroup = NULL; + while (1) { + pIter = mnodeGetNextVgroup(pIter, &pVgroup); + if (pVgroup == NULL) break; + if (pVgroup->pDb == pDb) { + mnodeSendSyncVgroupMsg(pVgroup); + } + mnodeDecVgroupRef(pVgroup); + } + + mLInfo("db:%s, is synced by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); + + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeProcessSyncDbMsg(SMnodeMsg *pMsg) { + SSyncDbMsg *pSyncDb = pMsg->rpcMsg.pCont; + mDebug("db:%s, syncdb is received from thandle:%p, ignore:%d", pSyncDb->db, pMsg->rpcMsg.handle, pSyncDb->ignoreNotExists); + + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pSyncDb->db); + if (pMsg->pDb == NULL) { + if (pSyncDb->ignoreNotExists) { + mDebug("db:%s, db is not exist, treat as success", pSyncDb->db); + return TSDB_CODE_SUCCESS; + } else { + mError("db:%s, failed to sync, invalid db", pSyncDb->db); + return TSDB_CODE_MND_INVALID_DB; + } + } + + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { + mError("db:%s, status:%d, in dropping", pSyncDb->db, pMsg->pDb->status); + return TSDB_CODE_MND_DB_IN_DROPPING; + } + + return mnodeSyncDb(pMsg->pDb, pMsg); +} + void mnodeDropAllDbs(SAcctObj *pAcct) { int32_t numOfDbs = 0; SDbObj *pDb = NULL; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 792e41dd5be3b2e0e0ce0fb1847126caf7dd3b94..db03da4fe1752bb0d2c4a380530c205d3a953339 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -39,7 +39,7 @@ #include "mnodeCluster.h" int32_t tsAccessSquence = 0; -int64_t tsDnodeRid = -1; +int64_t tsDnodeRid = -1; static void * tsDnodeSdb = NULL; static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; @@ -148,7 +148,7 @@ static int32_t mnodeDnodeActionDecode(SSdbRow *pRow) { } static int32_t mnodeDnodeActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); + int64_t numOfRows = sdbGetNumOfRows(tsDnodeSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create dnode:%s", tsLocalEp); mnodeCreateDnode(tsLocalEp, NULL); @@ -165,7 +165,7 @@ static int32_t mnodeDnodeActionRestored() { int32_t mnodeInitDnodes() { SDnodeObj tObj; - tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsDnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); pthread_mutex_init(&tsDnodeEpsMutex, NULL); SSdbTableDesc desc = { @@ -173,7 +173,7 @@ int32_t mnodeInitDnodes() { .name = "dnodes", .hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE, .maxRowSize = tsDnodeUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_AUTO, .fpInsert = mnodeDnodeActionInsert, .fpDelete = mnodeDnodeActionDelete, @@ -227,7 +227,7 @@ void mnodeCancelGetNextDnode(void *pIter) { } int32_t mnodeGetDnodesNum() { - return sdbGetNumOfRows(tsDnodeSdb); + return (int32_t)sdbGetNumOfRows(tsDnodeSdb); } int32_t mnodeGetOnlinDnodesCpuCoreNum() { @@ -263,6 +263,28 @@ int32_t mnodeGetOnlineDnodesNum() { return onlineDnodes; } +void mnodeGetOnlineAndTotalDnodesNum(int32_t *onlineNum, int32_t *totalNum) { + SDnodeObj *pDnode = NULL; + void * pIter = NULL; + int32_t onlineDnodes = 0, totalDnodes = 0; + + while (1) { + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + if (pDnode->status != TAOS_DN_STATUS_OFFLINE) ++onlineDnodes; + ++totalDnodes; + mnodeDecDnodeRef(pDnode); + } + + if (onlineNum) { + *onlineNum = onlineDnodes; + } + + if (totalNum) { + *totalNum = totalDnodes; + } +} + void *mnodeGetDnode(int32_t dnodeId) { return sdbGetRow(tsDnodeSdb, &dnodeId); } @@ -407,7 +429,7 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { int64_t checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && (checkTime != clusterCfg->checkTime)) { mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg parameters inconsistent", clusterCfg->timezone, @@ -638,9 +660,9 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { char *temp = strchr(dnodeEp, ':'); if (!temp) { - int len = strlen(dnodeEp); + int32_t len = (int32_t)strlen(dnodeEp); if (dnodeEp[len - 1] == ';') dnodeEp[len - 1] = 0; - len = strlen(dnodeEp); + len = (int32_t)strlen(dnodeEp); snprintf(dnodeEp + len, TSDB_EP_LEN - len, ":%d", tsServerPort); } ep = dnodeEp; @@ -700,6 +722,10 @@ int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) { static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); if (pDnode == NULL) { + if (strspn(ep, "0123456789 ;") != strlen(ep)) { + return TSDB_CODE_MND_DNODE_NOT_EXIST; + } + int32_t dnodeId = (int32_t)strtol(ep, NULL, 10); pDnode = mnodeGetDnode(dnodeId); if (pDnode == NULL) { diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 3ea41c41c6ffe89671853387e59733f7db7506cd..49473d3e06f54e1e66f507957d6351162990e030 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -136,14 +136,14 @@ int32_t mnodeInitMnodes() { mnodeMnodeInitLock(); SMnodeObj tObj; - tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsMnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_MNODE, .name = "mnodes", .hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE, .maxRowSize = tsMnodeUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_INT, .fpInsert = mnodeMnodeActionInsert, .fpDelete = mnodeMnodeActionDelete, @@ -176,7 +176,7 @@ void mnodeCleanupMnodes() { } int32_t mnodeGetMnodesNum() { - return sdbGetNumOfRows(tsMnodeSdb); + return (int32_t)sdbGetNumOfRows(tsMnodeSdb); } void *mnodeGetMnode(int32_t mnodeId) { diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index aaf8b694279299215dbfe386755c43ed2200e555..9bd8d7e4d7b0723ac4c5e4248e3c1193dfc15ceb 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -17,7 +17,6 @@ #include "os.h" #include "taoserror.h" #include "tsched.h" -#include "tsystem.h" #include "tutil.h" #include "tgrant.h" #include "tbn.h" diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 7c35829f88f4e7d1753a3ee00b13a9f87d03ae48..17a4282d05935684df4ab6585fef5f2398a62979 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -356,7 +356,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; - strcpy(pSchema[cols].name, "time(us)"); + strcpy(pSchema[cols].name, "time"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -385,12 +385,22 @@ static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, v SConnObj *pConnObj = NULL; int32_t cols = 0; char * pWrite; + void * pIter; char str[TSDB_IPv4ADDR_LEN + 6] = {0}; while (numOfRows < rows) { - pShow->pIter = mnodeGetNextConn(pShow->pIter, &pConnObj); - if (pConnObj == NULL) break; + pIter = mnodeGetNextConn(pShow->pIter, &pConnObj); + if (pConnObj == NULL) { + pShow->pIter = pIter; + break; + } + + if (numOfRows + pConnObj->numOfQueries >= rows) { + mnodeCancelGetNextConn(pIter); + break; + } + pShow->pIter = pIter; for (int32_t i = 0; i < pConnObj->numOfQueries; ++i) { SQueryDesc *pDesc = pConnObj->pQueries + i; cols = 0; @@ -518,12 +528,22 @@ static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, v SConnObj *pConnObj = NULL; int32_t cols = 0; char * pWrite; + void * pIter; char ipStr[TSDB_IPv4ADDR_LEN + 6]; while (numOfRows < rows) { - pShow->pIter = mnodeGetNextConn(pShow->pIter, &pConnObj); - if (pConnObj == NULL) break; + pIter = mnodeGetNextConn(pShow->pIter, &pConnObj); + if (pConnObj == NULL) { + pShow->pIter = pIter; + break; + } + + if (numOfRows + pConnObj->numOfStreams >= rows) { + mnodeCancelGetNextConn(pIter); + break; + } + pShow->pIter = pIter; for (int32_t i = 0; i < pConnObj->numOfStreams; ++i) { SStreamDesc *pDesc = pConnObj->pStreams + i; cols = 0; diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c index 200f589b7866d3fa6cf9865be397935657d7ba86..dadd8155d8648244dba762afec63bfc6f2ac3721 100644 --- a/src/mnode/src/mnodeRead.c +++ b/src/mnode/src/mnodeRead.c @@ -50,6 +50,9 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + if (!epSet) { + return TSDB_CODE_MND_OUT_OF_MEMORY; + } mnodeGetMnodeEpSetForShell(epSet, true); rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index ae495108b3a3d0afd479057d9828ae4c69d4537f..381cb11952dbdd639408bda1efdae258a9a94a30 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -35,7 +35,7 @@ #include "mnodeSdb.h" #define SDB_TABLE_LEN 12 -#define MAX_QUEUED_MSG_NUM 10000 +#define MAX_QUEUED_MSG_NUM 100000 typedef enum { SDB_ACTION_INSERT = 0, @@ -207,7 +207,7 @@ static void sdbRestoreTables() { (*pTable->fpRestored)(); } - totalRows += pTable->numOfRows; + totalRows += (int32_t)pTable->numOfRows; numOfTables++; sdbInfo("vgId:1, sdb:%s is checked, rows:%" PRId64, pTable->name, pTable->numOfRows); } @@ -242,11 +242,6 @@ void sdbUpdateMnodeRoles() { mnodeUpdateMnodeEpSet(NULL); } -static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { - sdbUpdateMnodeRoles(); - return 0; -} - static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { return walGetWalFile(tsSdbMgmt.wal, fileName, fileId); } @@ -262,7 +257,9 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) { sdbUpdateMnodeRoles(); } -static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; } +static void sdbStartFileSync(int32_t vgId) {} + +static void sdbStopFileSync(int32_t vgId, uint64_t fversion) {} static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {} @@ -396,14 +393,14 @@ int32_t sdbUpdateSync(void *pMnodes) { syncInfo.version = sdbGetVersion(); syncInfo.syncCfg = syncCfg; sprintf(syncInfo.path, "%s", tsMnodeDir); - syncInfo.getFileInfo = sdbGetFileInfo; - syncInfo.getWalInfo = sdbGetWalInfo; - syncInfo.writeToCache = sdbWriteFwdToQueue; + syncInfo.getWalInfoFp = sdbGetWalInfo; + syncInfo.writeToCacheFp = sdbWriteFwdToQueue; syncInfo.confirmForward = sdbConfirmForward; - syncInfo.notifyRole = sdbNotifyRole; - syncInfo.notifyFileSynced = sdbNotifyFileSynced; - syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl; - syncInfo.getVersion = sdbGetSyncVersion; + syncInfo.notifyRoleFp = sdbNotifyRole; + syncInfo.startSyncFileFp = sdbStartFileSync; + syncInfo.stopSyncFileFp = sdbStopFileSync; + syncInfo.notifyFlowCtrlFp = sdbNotifyFlowCtrl; + syncInfo.getVersionFp = sdbGetSyncVersion; tsSdbMgmt.cfg = syncCfg; if (tsSdbMgmt.sync) { @@ -475,7 +472,7 @@ void sdbIncRef(void *tparam, void *pRow) { if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos); int32_t refCount = atomic_add_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); } @@ -484,11 +481,11 @@ void sdbDecRef(void *tparam, void *pRow) { if (pRow == NULL || tparam == NULL) return; SSdbTable *pTable = tparam; - int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos); + int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos); int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); - int32_t *updateEnd = pRow + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char *)pRow + pTable->refCountPos - 4); if (refCount <= 0 && *updateEnd) { sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount); SSdbRow row = {.pObj = pRow}; @@ -501,7 +498,7 @@ static void *sdbGetRowMeta(SSdbTable *pTable, void *key) { int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } void **ppRow = (void **)taosHashGet(pTable->iHandle, key, keySize); @@ -534,7 +531,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } pthread_mutex_lock(&pTable->mutex); @@ -555,7 +552,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { int32_t code = (*pTable->fpInsert)(pRow); if (code != TSDB_CODE_SUCCESS) { - sdbError("vgId:1, sdb:%s, failed to insert key:%s to hash, remove it", pTable->name, + sdbError("vgId:1, sdb:%s, failed to perform insert action for key:%s, remove it", pTable->name, sdbGetRowStr(pTable, pRow->pObj)); sdbDeleteHash(pTable, pRow); } @@ -564,7 +561,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { } static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { - int32_t *updateEnd = pRow->pObj + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char*)pRow->pObj + pTable->refCountPos - 4); bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; if (!set) { sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->name, @@ -577,7 +574,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { void * key = sdbGetObjKey(pTable, pRow->pObj); int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); + keySize = (int32_t)strlen((char *)key); } pthread_mutex_lock(&pTable->mutex); @@ -683,7 +680,7 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * if (pRow != NULL) { // forward to peers pRow->processedCount = 0; - int32_t syncCode = syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC); + int32_t syncCode = syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC, false); if (syncCode <= 0) pRow->processedCount = 1; if (syncCode < 0) { @@ -703,7 +700,7 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync - syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC); + syncForwardToPeer(tsSdbMgmt.sync, pHead, pRow, TAOS_QTYPE_RPC, false); // from wal or forward msg, row not created, should add into hash if (action == SDB_ACTION_INSERT) { @@ -764,7 +761,7 @@ bool sdbCheckRowDeleted(void *tparam, void *pRow) { SSdbTable *pTable = tparam; if (pTable == NULL) return false; - int32_t *updateEnd = pRow + pTable->refCountPos - 4; + int32_t *updateEnd = (int32_t *)((char*)pRow + pTable->refCountPos - 4); return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1; } @@ -942,14 +939,14 @@ static int32_t sdbInitWorker() { static void sdbCleanupWorker() { for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsSdbWQset); } } for (int32_t i = 0; i < tsSdbPool.num; ++i) { SSdbWorker *pWorker = tsSdbPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } @@ -1122,7 +1119,7 @@ static void *sdbWorkerFp(void *pWorker) { sdbConfirmForward(1, pRow, pRow->code); } else { if (qtype == TAOS_QTYPE_FWD) { - syncConfirmForward(tsSdbMgmt.sync, pRow->pHead.version, pRow->code); + syncConfirmForward(tsSdbMgmt.sync, pRow->pHead.version, pRow->code, false); } sdbFreeFromQueue(pRow); } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 6b9f0e26a762092cfae02b5724d6cfedd24194d2..03772f2724c87c74da2f3ab7128705d90c5bb1ad 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -109,6 +109,7 @@ static char *mnodeGetShowType(int32_t showType) { case TSDB_MGMT_TABLE_VNODES: return "show vnodes"; case TSDB_MGMT_TABLE_CLUSTER: return "show clusters"; case TSDB_MGMT_TABLE_STREAMTABLES : return "show streamtables"; + case TSDB_MGMT_TABLE_TP: return "show topics"; default: return "undefined"; } } @@ -218,7 +219,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) { } pRsp->numOfRows = htonl(rowsRead); - pRsp->precision = htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision + pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision pMsg->rpcRsp.rsp = pRsp; pMsg->rpcRsp.len = size; @@ -280,8 +281,11 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { } } - pRsp->onlineDnodes = htonl(mnodeGetOnlineDnodesNum()); - pRsp->totalDnodes = htonl(mnodeGetDnodesNum()); + int32_t onlineDnodes = 0, totalDnodes = 0; + mnodeGetOnlineAndTotalDnodesNum(&onlineDnodes, &totalDnodes); + + pRsp->onlineDnodes = htonl(onlineDnodes); + pRsp->totalDnodes = htonl(totalDnodes); mnodeGetMnodeEpSetForShell(&pRsp->epSet, false); pMsg->rpcRsp.rsp = pRsp; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index de37b0934539a512630c008fc1b1245104dd7483..39eca8819d8ace50b6fa82fadb1d4464247568d6 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -68,7 +68,7 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, static int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); - + static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg); @@ -108,10 +108,12 @@ static int32_t mnodeChildTableActionDestroy(SSdbRow *pRow) { static int32_t mnodeChildTableActionInsert(SSdbRow *pRow) { SCTableObj *pTable = pRow->pObj; + int32_t code = 0; SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("ctable:%s, not in vgId:%d", pTable->info.tableId, pTable->vgId); + code = -1; } SDbObj *pDb = NULL; @@ -119,6 +121,7 @@ static int32_t mnodeChildTableActionInsert(SSdbRow *pRow) { pDb = mnodeGetDb(pVgroup->dbName); if (pDb == NULL) { mError("ctable:%s, vgId:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); + code = -1; } } @@ -127,6 +130,7 @@ static int32_t mnodeChildTableActionInsert(SSdbRow *pRow) { pAcct = mnodeGetAcct(pDb->acct); if (pAcct == NULL) { mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->acct); + code = -1; } } @@ -139,6 +143,7 @@ static int32_t mnodeChildTableActionInsert(SSdbRow *pRow) { if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1); } else { mError("table:%s:%p, correspond stable not found suid:%" PRIu64, pTable->info.tableId, pTable, pTable->suid); + code = -1; } } else { grantAdd(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1); @@ -146,25 +151,38 @@ static int32_t mnodeChildTableActionInsert(SSdbRow *pRow) { } if (pDb) mnodeAddTableIntoDb(pDb); - if (pVgroup) mnodeAddTableIntoVgroup(pVgroup, pTable); + if (pVgroup) { + if (mnodeAddTableIntoVgroup(pVgroup, pTable, pRow->pMsg == NULL) != 0) { + mError("table:%s, vgId:%d tid:%d, failed to perform insert action, uid:%" PRIu64 " suid:%" PRIu64, + pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, pTable->suid); + code = -1; + } + } mnodeDecVgroupRef(pVgroup); mnodeDecDbRef(pDb); mnodeDecAcctRef(pAcct); - return TSDB_CODE_SUCCESS; + if (code == 0) { + mTrace("table:%s, vgId:%d tid:%d, perform insert action, uid:%" PRIu64 " suid:%" PRIu64, pTable->info.tableId, + pTable->vgId, pTable->tid, pTable->uid, pTable->suid); + } + + return code; } static int32_t mnodeChildTableActionDelete(SSdbRow *pRow) { SCTableObj *pTable = pRow->pObj; if (pTable->vgId == 0) { + mError("table:%s, vgId:%d tid:%d, failed to perform delete action, uid:%" PRIu64 " suid:%" PRIu64, + pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, pTable->suid); return TSDB_CODE_MND_VGROUP_NOT_EXIST; } SVgObj *pVgroup = NULL; SDbObj *pDb = NULL; SAcctObj *pAcct = NULL; - + pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup != NULL) pDb = mnodeGetDb(pVgroup->dbName); if (pDb != NULL) pAcct = mnodeGetAcct(pDb->acct); @@ -180,14 +198,16 @@ static int32_t mnodeChildTableActionDelete(SSdbRow *pRow) { grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1); if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1); } - + if (pDb != NULL) mnodeRemoveTableFromDb(pDb); if (pVgroup != NULL) mnodeRemoveTableFromVgroup(pVgroup, pTable); mnodeDecVgroupRef(pVgroup); mnodeDecDbRef(pDb); mnodeDecAcctRef(pAcct); - + + mTrace("table:%s, vgId:%d tid:%d, perform delete action, uid:%" PRIu64 " suid:%" PRIu64, pTable->info.tableId, + pTable->vgId, pTable->tid, pTable->uid, pTable->suid); return TSDB_CODE_SUCCESS; } @@ -195,19 +215,19 @@ static int32_t mnodeChildTableActionUpdate(SSdbRow *pRow) { SCTableObj *pNew = pRow->pObj; SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); if (pTable != pNew) { - void *oldTableId = pTable->info.tableId; + void *oldTableId = pTable->info.tableId; void *oldSql = pTable->sql; void *oldSchema = pTable->schema; void *oldSTable = pTable->superTable; int32_t oldRefCount = pTable->refCount; - + memcpy(pTable, pNew, sizeof(SCTableObj)); - + pTable->refCount = oldRefCount; pTable->sql = pNew->sql; pTable->schema = pNew->schema; pTable->superTable = oldSTable; - + free(pNew); free(oldSql); free(oldSchema); @@ -222,23 +242,23 @@ static int32_t mnodeChildTableActionEncode(SSdbRow *pRow) { SCTableObj *pTable = pRow->pObj; assert(pTable != NULL && pRow->rowData != NULL); - int32_t len = strlen(pTable->info.tableId); + int32_t len = (int32_t)strlen(pTable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pRow->rowData, pTable->info.tableId, len); - memset(pRow->rowData + len, 0, 1); + memset((char *)pRow->rowData + len, 0, 1); len++; - memcpy(pRow->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize); + memcpy((char *)pRow->rowData + len, (char *)pTable + sizeof(char *), tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema); - memcpy(pRow->rowData + len, pTable->schema, schemaSize); + memcpy((char *)pRow->rowData + len, pTable->schema, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { - memcpy(pRow->rowData + len, pTable->sql, pTable->sqlLen); + memcpy((char *)pRow->rowData + len, pTable->sql, pTable->sqlLen); len += pTable->sqlLen; } } @@ -253,7 +273,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pRow->rowData); + int32_t len = (int32_t)strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN) { free(pTable); return TSDB_CODE_MND_INVALID_TABLE_ID; @@ -261,7 +281,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { pTable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pTable + sizeof(char *), pRow->rowData + len, tsChildTableUpdateSize); + memcpy((char *)pTable + sizeof(char *), (char *)pRow->rowData + len, tsChildTableUpdateSize); len += tsChildTableUpdateSize; if (pTable->info.type != TSDB_CHILD_TABLE) { @@ -271,7 +291,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_TYPE; } - memcpy(pTable->schema, pRow->rowData + len, schemaSize); + memcpy(pTable->schema, (char *)pRow->rowData + len, schemaSize); len += schemaSize; if (pTable->sqlLen != 0) { @@ -280,7 +300,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) { mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_OUT_OF_MEMORY; } - memcpy(pTable->sql, pRow->rowData + len, pTable->sqlLen); + memcpy(pTable->sql, (char *)pRow->rowData + len, pTable->sqlLen); } } @@ -297,7 +317,7 @@ static int32_t mnodeChildTableActionRestored() { pIter = mnodeGetNextChildTable(pIter, &pTable); if (pTable == NULL) break; - SDbObj *pDb = mnodeGetDbByTableId(pTable->info.tableId); + SDbObj *pDb = mnodeGetDbByTableName(pTable->info.tableId); if (pDb == NULL || pDb->status != TSDB_DB_STATUS_READY) { mError("ctable:%s, failed to get db or db in dropping, discard it", pTable->info.tableId); SSdbRow desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .pTable = tsChildTableSdb}; @@ -352,14 +372,14 @@ static int32_t mnodeChildTableActionRestored() { static int32_t mnodeInitChildTables() { SCTableObj tObj; - tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; + tsChildTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type); SSdbTableDesc desc = { .id = SDB_TABLE_CTABLE, .name = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_VAR_STRING, .fpInsert = mnodeChildTableActionInsert, .fpDelete = mnodeChildTableActionDelete, @@ -399,13 +419,13 @@ static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) { if (pStable->vgHash == NULL) { pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - mDebug("table:%s, create hash:%p", pStable->info.tableId, pStable->vgHash); + mDebug("stable:%s, create vgId hash:%p", pStable->info.tableId, pStable->vgHash); } if (pStable->vgHash != NULL) { if (taosHashGet(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)) == NULL) { taosHashPut(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId)); - mDebug("table:%s, vgId:%d is put into stable hash:%p, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, + mDebug("stable:%s, vgId:%d is put into stable vgId hash:%p, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, pStable->vgHash, taosHashGetSize(pStable->vgHash)); } } @@ -443,19 +463,21 @@ static int32_t mnodeSuperTableActionDestroy(SSdbRow *pRow) { static int32_t mnodeSuperTableActionInsert(SSdbRow *pRow) { SSTableObj *pStable = pRow->pObj; - SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); + SDbObj * pDb = mnodeGetDbByTableName(pStable->info.tableId); if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) { mnodeAddSuperTableIntoDb(pDb); } mnodeDecDbRef(pDb); taosHashPut(tsSTableUidHash, &pStable->uid, sizeof(int64_t), &pStable, sizeof(int64_t)); + + mTrace("stable:%s, perform insert action, uid:%" PRIu64, pStable->info.tableId, pStable->uid); return TSDB_CODE_SUCCESS; } static int32_t mnodeSuperTableActionDelete(SSdbRow *pRow) { SSTableObj *pStable = pRow->pObj; - SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); + SDbObj * pDb = mnodeGetDbByTableName(pStable->info.tableId); if (pDb != NULL) { mnodeRemoveSuperTableFromDb(pDb); mnodeDropAllChildTablesInStable((SSTableObj *)pStable); @@ -463,6 +485,8 @@ static int32_t mnodeSuperTableActionDelete(SSdbRow *pRow) { mnodeDecDbRef(pDb); taosHashRemove(tsSTableUidHash, &pStable->uid, sizeof(int64_t)); + + mTrace("stable:%s, perform delete action, uid:%" PRIu64, pStable->info.tableId, pStable->uid); return TSDB_CODE_SUCCESS; } @@ -501,18 +525,18 @@ static int32_t mnodeSuperTableActionEncode(SSdbRow *pRow) { SSTableObj *pStable = pRow->pObj; assert(pRow->pObj != NULL && pRow->rowData != NULL); - int32_t len = strlen(pStable->info.tableId); + int32_t len = (int32_t)strlen(pStable->info.tableId); if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pRow->rowData, pStable->info.tableId, len); - memset(pRow->rowData + len, 0, 1); + memset((char *)pRow->rowData + len, 0, 1); len++; - memcpy(pRow->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize); + memcpy((char *)pRow->rowData + len, (char *)pStable + sizeof(char *), tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); - memcpy(pRow->rowData + len, pStable->schema, schemaSize); + memcpy((char *)pRow->rowData + len, pStable->schema, schemaSize); len += schemaSize; pRow->rowSize = len; @@ -525,7 +549,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; - int32_t len = strlen(pRow->rowData); + int32_t len = (int32_t)strlen(pRow->rowData); if (len >= TSDB_TABLE_FNAME_LEN){ free(pStable); return TSDB_CODE_MND_INVALID_TABLE_ID; @@ -533,7 +557,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { pStable->info.tableId = strdup(pRow->rowData); len++; - memcpy((char*)pStable + sizeof(char *), pRow->rowData + len, tsSuperTableUpdateSize); + memcpy((char *)pStable + sizeof(char *), (char *)pRow->rowData + len, tsSuperTableUpdateSize); len += tsSuperTableUpdateSize; int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags); @@ -543,8 +567,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) { return TSDB_CODE_MND_NOT_SUPER_TABLE; } - memcpy(pStable->schema, pRow->rowData + len, schemaSize); - + memcpy(pStable->schema, (char *)pRow->rowData + len, schemaSize); pRow->pObj = pStable; return TSDB_CODE_SUCCESS; @@ -556,14 +579,14 @@ static int32_t mnodeSuperTableActionRestored() { static int32_t mnodeInitSuperTables() { SSTableObj tObj; - tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; + tsSuperTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type); SSdbTableDesc desc = { .id = SDB_TABLE_STABLE, .name = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_VAR_STRING, .fpInsert = mnodeSuperTableActionInsert, .fpDelete = mnodeSuperTableActionDelete, @@ -611,7 +634,7 @@ int32_t mnodeInitTables() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_ALTER_TABLE, mnodeProcessAlterTableMsg); mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_TABLE_META, mnodeProcessTableMetaMsg); mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_STABLE_VGROUP, mnodeProcessSuperTableVgroupMsg); - + mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_TABLE_RSP, mnodeProcessCreateChildTableRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_TABLE_RSP, mnodeProcessDropChildTableRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_STABLE_RSP, mnodeProcessDropSuperTableRsp); @@ -748,9 +771,12 @@ void mnodeDestroySubMsg(SMnodeMsg *pSubMsg) { } static int32_t mnodeValidateCreateTableMsg(SCreateTableMsg *pCreateTable, SMnodeMsg *pMsg) { - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pCreateTable->db); if (pMsg->pDb == NULL) { - mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + pMsg->pDb = mnodeGetDbByTableName(pCreateTable->tableName); + } + + if (pMsg->pDb == NULL) { + mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableName); return TSDB_CODE_MND_DB_NOT_SELECTED; } @@ -759,28 +785,28 @@ static int32_t mnodeValidateCreateTableMsg(SCreateTableMsg *pCreateTable, SMnode return TSDB_CODE_MND_DB_IN_DROPPING; } - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreateTable->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreateTable->tableName); if (pMsg->pTable != NULL && pMsg->retry == 0) { if (pCreateTable->getMeta) { - mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableName); return mnodeGetChildTableMeta(pMsg); } else if (pCreateTable->igExists) { - mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableId); + mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, pCreateTable->tableName); return TSDB_CODE_SUCCESS; } else { mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, - pCreateTable->tableId); + pCreateTable->tableName); return TSDB_CODE_MND_TABLE_ALREADY_EXIST; } } if (pCreateTable->numOfTags != 0) { - mDebug("msg:%p, app:%p table:%s, create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - pCreateTable->tableId, pMsg->rpcMsg.handle); + mDebug("msg:%p, app:%p table:%s, batch create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableName, pMsg->rpcMsg.handle); return mnodeProcessCreateSuperTableMsg(pMsg); } else { - mDebug("msg:%p, app:%p table:%s, create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - pCreateTable->tableId, pMsg->rpcMsg.handle); + mDebug("msg:%p, app:%p table:%s, batch create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, + pCreateTable->tableName, pMsg->rpcMsg.handle); return mnodeProcessCreateChildTableMsg(pMsg); } } @@ -830,17 +856,19 @@ static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) { return code; } else if (code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; mnodeDestroySubMsg(pMsg); } if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { - dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, pMsg->pBatchMasterMsg->code); } return TSDB_CODE_MND_ACTION_IN_PROGRESS; } else { // batch master replay, reprocess the whole batch assert(0); + return TSDB_CODE_MND_MSG_NOT_PROCESSED; } } } @@ -860,49 +888,51 @@ static int32_t mnodeProcessCreateTableMsg(SMnodeMsg *pMsg) { } SCreateTableMsg *p = (SCreateTableMsg*)((char*) pCreate + sizeof(SCMCreateTableMsg)); - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(p->db); if (pMsg->pDb == NULL) { - mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, p->tableId); + pMsg->pDb = mnodeGetDbByTableName(p->tableName); + } + + if (pMsg->pDb == NULL) { + mError("msg:%p, app:%p table:%s, failed to create, db not selected", pMsg, pMsg->rpcMsg.ahandle, p->tableName); return TSDB_CODE_MND_DB_NOT_SELECTED; } - + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); return TSDB_CODE_MND_DB_IN_DROPPING; } - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(p->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(p->tableName); if (pMsg->pTable != NULL && pMsg->retry == 0) { if (p->getMeta) { - mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, p->tableId); + mDebug("msg:%p, app:%p table:%s, continue to get meta", pMsg, pMsg->rpcMsg.ahandle, p->tableName); return mnodeGetChildTableMeta(pMsg); } else if (p->igExists) { - mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, p->tableId); + mDebug("msg:%p, app:%p table:%s, is already exist", pMsg, pMsg->rpcMsg.ahandle, p->tableName); return TSDB_CODE_SUCCESS; } else { - mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, - p->tableId); + mError("msg:%p, app:%p table:%s, failed to create, table already exist", pMsg, pMsg->rpcMsg.ahandle, p->tableName); return TSDB_CODE_MND_TABLE_ALREADY_EXIST; } } if (p->numOfTags != 0) { mDebug("msg:%p, app:%p table:%s, create stable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - p->tableId, pMsg->rpcMsg.handle); + p->tableName, pMsg->rpcMsg.handle); return mnodeProcessCreateSuperTableMsg(pMsg); } else { mDebug("msg:%p, app:%p table:%s, create ctable msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - p->tableId, pMsg->rpcMsg.handle); + p->tableName, pMsg->rpcMsg.handle); return mnodeProcessCreateChildTableMsg(pMsg); } } static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { SCMDropTableMsg *pDrop = pMsg->rpcMsg.pCont; - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pDrop->tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(pDrop->name); if (pMsg->pDb == NULL) { mError("msg:%p, app:%p table:%s, failed to drop table, db not selected or db in dropping", pMsg, - pMsg->rpcMsg.ahandle, pDrop->tableId); + pMsg->rpcMsg.ahandle, pDrop->name); return TSDB_CODE_MND_DB_NOT_SELECTED; } @@ -911,19 +941,21 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_DB_IN_DROPPING; } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("msg:%p, app:%p table:%s, failed to drop table, in monitor database", pMsg, pMsg->rpcMsg.ahandle, - pDrop->tableId); + pDrop->name); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->name); if (pMsg->pTable == NULL) { if (pDrop->igNotExists) { - mDebug("msg:%p, app:%p table:%s is not exist, treat as success", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId); + mDebug("msg:%p, app:%p table:%s is not exist, treat as success", pMsg, pMsg->rpcMsg.ahandle, pDrop->name); return TSDB_CODE_SUCCESS; } else { - mError("msg:%p, app:%p table:%s, failed to drop, table not exist", pMsg, pMsg->rpcMsg.ahandle, pDrop->tableId); + mError("msg:%p, app:%p table:%s, failed to drop, table not exist", pMsg, pMsg->rpcMsg.ahandle, pDrop->name); return TSDB_CODE_MND_INVALID_TABLE_NAME; } } @@ -931,12 +963,12 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable->type == TSDB_SUPER_TABLE) { SSTableObj *pSTable = (SSTableObj *)pMsg->pTable; mInfo("msg:%p, app:%p table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", pMsg, - pMsg->rpcMsg.ahandle, pDrop->tableId, pSTable->uid, pSTable->numOfTables, taosHashGetSize(pSTable->vgHash)); + pMsg->rpcMsg.ahandle, pDrop->name, pSTable->uid, pSTable->numOfTables, taosHashGetSize(pSTable->vgHash)); return mnodeProcessDropSuperTableMsg(pMsg); } else { SCTableObj *pCTable = (SCTableObj *)pMsg->pTable; mInfo("msg:%p, app:%p table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, - pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid); + pDrop->name, pCTable->vgId, pCTable->tid, pCTable->uid); return mnodeProcessDropChildTableMsg(pMsg); } } @@ -945,29 +977,29 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; pInfo->createFlag = htons(pInfo->createFlag); mDebug("msg:%p, app:%p table:%s, table meta msg is received from thandle:%p, createFlag:%d", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId, pMsg->rpcMsg.handle, pInfo->createFlag); + pInfo->tableFname, pMsg->rpcMsg.handle, pInfo->createFlag); - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pInfo->tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(pInfo->tableFname); if (pMsg->pDb == NULL) { mError("msg:%p, app:%p table:%s, failed to get table meta, db not selected", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId); + pInfo->tableFname); return TSDB_CODE_MND_DB_NOT_SELECTED; } - + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); return TSDB_CODE_MND_DB_IN_DROPPING; } - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pInfo->tableFname); if (pMsg->pTable == NULL) { if (!pInfo->createFlag) { mError("msg:%p, app:%p table:%s, failed to get table meta, table not exist", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId); + pInfo->tableFname); return TSDB_CODE_MND_INVALID_TABLE_NAME; } else { mDebug("msg:%p, app:%p table:%s, failed to get table meta, start auto create table ", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId); + pInfo->tableFname); return mnodeAutoCreateChildTable(pMsg); } } else { @@ -1007,12 +1039,12 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { SSTableObj * pStable = calloc(1, sizeof(SSTableObj)); if (pStable == NULL) { - mError("msg:%p, app:%p table:%s, failed to create, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, no enough memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return TSDB_CODE_MND_OUT_OF_MEMORY; } int64_t us = taosGetTimestampUs(); - pStable->info.tableId = strdup(pCreate->tableId); + pStable->info.tableId = strdup(pCreate->tableName); pStable->info.type = TSDB_SUPER_TABLE; pStable->createdTime = taosGetTimestampMs(); pStable->uid = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); @@ -1026,41 +1058,27 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { pStable->schema = (SSchema *)calloc(1, schemaSize); if (pStable->schema == NULL) { free(pStable); - mError("msg:%p, app:%p table:%s, failed to create, no schema input", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, no schema input", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return TSDB_CODE_MND_INVALID_TABLE_NAME; } memcpy(pStable->schema, pCreate->schema, numOfCols * sizeof(SSchema)); if (pStable->numOfColumns > TSDB_MAX_COLUMNS || pStable->numOfTags > TSDB_MAX_TAGS) { - mError("msg:%p, app:%p table:%s, failed to create, too many columns", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, too many columns", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return TSDB_CODE_MND_INVALID_TABLE_NAME; } pStable->nextColId = 0; - // TODO extract method to valid the schema - int32_t schemaLen = 0; - int32_t tagLen = 0; for (int32_t col = 0; col < numOfCols; col++) { SSchema *tschema = pStable->schema; tschema[col].colId = pStable->nextColId++; tschema[col].bytes = htons(tschema[col].bytes); - - if (col < pStable->numOfTables) { - schemaLen += tschema[col].bytes; - } else { - tagLen += tschema[col].bytes; - } - - if (!isValidDataType(tschema[col].type)) { - mError("msg:%p, app:%p table:%s, failed to create, invalid data type in schema", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); - return TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG; - } } - if (schemaLen > (TSDB_MAX_BYTES_PER_ROW || tagLen > TSDB_MAX_TAGS_LEN)) { - mError("msg:%p, app:%p table:%s, failed to create, schema is too long", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + if (!tIsValidSchema(pStable->schema, pStable->numOfColumns, pStable->numOfTags)) { + mError("msg:%p, app:%p table:%s, failed to create table, invalid schema", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG; } @@ -1080,7 +1098,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeDestroySuperTable(pStable); pMsg->pTable = NULL; - mError("msg:%p, app:%p table:%s, failed to create, sdb error", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to create, sdb error", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); } return code; @@ -1090,20 +1108,13 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { SSTableObj *pTable = (SSTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { mError("msg:%p, app:%p stable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); - } else { - mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); + + return code; } - return code; -} - -static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { - if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; + mLInfo("msg:%p, app:%p stable:%s, is dropped from sdb", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); SSTableObj *pStable = (SSTableObj *)pMsg->pTable; - mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle, - pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash)); - if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL); while (pVgId) { @@ -1115,7 +1126,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { pDrop->contLen = htonl(sizeof(SDropSTableMsg)); pDrop->vgId = htonl(pVgroup->vgId); pDrop->uid = htobe64(pStable->uid); - mnodeExtractTableName(pStable->info.tableId, pDrop->tableId); + mnodeExtractTableName(pStable->info.tableId, pDrop->tableFname); mInfo("msg:%p, app:%p stable:%s, send drop stable msg to vgId:%d, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId, pVgroup->vgId, pStable->vgHash, @@ -1129,8 +1140,18 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { taosHashCancelIterate(pStable->vgHash, pVgId); mnodeDropAllChildTablesInStable(pStable); - } - + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { + if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; + + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; + mInfo("msg:%p, app:%p stable:%s will be dropped, hash:%p sizeOfVgList:%d", pMsg, pMsg->rpcMsg.ahandle, + pStable->info.tableId, pStable->vgHash, taosHashGetSize(pStable->vgHash)); + SSdbRow row = { .type = SDB_OPER_GLOBAL, .pTable = tsSuperTableSdb, @@ -1266,7 +1287,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c } // int32_t rowSize = 0; - uint32_t len = strlen(newTagName); + uint32_t len = (int32_t)strlen(newTagName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -1274,7 +1295,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c if (mnodeFindSuperTableTagIndex(pStable, newTagName) >= 0) { return TSDB_CODE_MND_TAG_ALREAY_EXIST; } - + // update SSchema *schema = (SSchema *) (pStable->schema + pStable->numOfColumns + col); tstrncpy(schema->name, newTagName, sizeof(schema->name)); @@ -1429,7 +1450,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char } // int32_t rowSize = 0; - uint32_t len = strlen(newName); + uint32_t len = (uint32_t)strlen(newName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -1437,7 +1458,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char if (mnodeFindSuperTableColumnIndex(pStable, newName) >= 0) { return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } - + // update SSchema *schema = (SSchema *) (pStable->schema + col); tstrncpy(schema->name, newName, sizeof(schema->name)); @@ -1460,7 +1481,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -1470,9 +1491,9 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, int32_t cols = 0; SSchema *pSchema = pMeta->schema; - SSchema tbnameSchema = tGetTableNameColumnSchema(); - pShow->bytes[cols] = tbnameSchema.bytes; - pSchema[cols].type = tbnameSchema.type; + SSchema* tbnameSchema = tGetTbnameColumnSchema(); + pShow->bytes[cols] = tbnameSchema->bytes; + pSchema[cols].type = tbnameSchema->type; strcpy(pSchema[cols].name, "name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -1525,7 +1546,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return 0; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -1534,12 +1555,12 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - prefixLen = strlen(prefix); + prefixLen = (int32_t)strlen(prefix); SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char stableName[TSDB_TABLE_NAME_LEN] = {0}; - while (numOfRows < rows) { + while (numOfRows < rows) { pShow->pIter = mnodeGetNextSuperTable(pShow->pIter, &pTable); if (pTable == NULL) break; if (strncmp(pTable->info.tableId, prefix, prefixLen)) { @@ -1558,11 +1579,11 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - - int16_t len = strnlen(stableName, TSDB_TABLE_NAME_LEN - 1); + + int16_t len = (int16_t)strnlen(stableName, TSDB_TABLE_NAME_LEN - 1); *(int16_t*) pWrite = len; pWrite += sizeof(int16_t); // todo refactor - + strncpy(pWrite, stableName, len); cols++; @@ -1602,7 +1623,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); mInfo("db:%s, all super tables will be dropped from sdb", pDropDb->name); @@ -1629,7 +1650,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; assert(numOfCols <= TSDB_MAX_COLUMNS); - + for (int32_t i = 0; i < numOfCols; ++i) { tstrncpy(pSchema->name, pTable->schema[i].name, sizeof(pSchema->name)); pSchema->type = pTable->schema[i].type; @@ -1655,7 +1676,7 @@ static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); pMeta->tableType = pTable->info.type; pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromSuperTable(pMeta->schema, pTable); - tstrncpy(pMeta->tableId, pTable->info.tableId, sizeof(pMeta->tableId)); + tstrncpy(pMeta->tableFname, pTable->info.tableId, sizeof(pMeta->tableFname)); pMsg->rpcRsp.len = pMeta->contLen; pMeta->contLen = htons(pMeta->contLen); @@ -1709,7 +1730,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; pVgroupMsg->numOfVgroups = 0; - + msg += sizeof(SVgroupsMsg); } else { SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; @@ -1755,9 +1776,9 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { rpcFreeCont(pRsp); return TSDB_CODE_MND_INVALID_TABLE_NAME; } else { - pRsp->numOfTables = htonl(pRsp->numOfTables); + pRsp->numOfTables = (int32_t)htonl(pRsp->numOfTables); pMsg->rpcRsp.rsp = pRsp; - pMsg->rpcRsp.len = msg - (char *)pRsp; + pMsg->rpcRsp.len = (int32_t)((char *)msg - (char *)pRsp); return TSDB_CODE_SUCCESS; } @@ -1798,7 +1819,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pCreateMsg, SCTabl return NULL; } - mnodeExtractTableName(pTable->info.tableId, pCreate->tableId); + mnodeExtractTableName(pTable->info.tableId, pCreate->tableFname); pCreate->contLen = htonl(contLen); pCreate->vgId = htonl(pTable->vgId); pCreate->tableType = pTable->info.type; @@ -1806,9 +1827,9 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pCreateMsg, SCTabl pCreate->tid = htonl(pTable->tid); pCreate->sqlDataLen = htonl(pTable->sqlLen); pCreate->uid = htobe64(pTable->uid); - + if (pTable->info.type == TSDB_CHILD_TABLE) { - mnodeExtractTableName(pTable->superTable->info.tableId, pCreate->superTableId); + mnodeExtractTableName(pTable->superTable->info.tableId, pCreate->stableFname); pCreate->numOfColumns = htons(pTable->superTable->numOfColumns); pCreate->numOfTags = htons(pTable->superTable->numOfTags); pCreate->sversion = htonl(pTable->superTable->sversion); @@ -1823,7 +1844,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pCreateMsg, SCTabl pCreate->tagDataLen = 0; pCreate->superTableUid = 0; } - + SSchema *pSchema = (SSchema *) pCreate->data; if (pTable->info.type == TSDB_CHILD_TABLE) { memcpy(pSchema, pTable->superTable->schema, totalCols * sizeof(SSchema)); @@ -1891,15 +1912,14 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { pMsg->rpcMsg.handle); if (pMsg->pBatchMasterMsg) { - ++pMsg->pBatchMasterMsg->successed; - if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received - >= pMsg->pBatchMasterMsg->expected) { - dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); - } + ++pMsg->pBatchMasterMsg->successed; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } - mnodeDestroySubMsg(pMsg); + mnodeDestroySubMsg(pMsg); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); @@ -1910,6 +1930,19 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { pMsg->rpcMsg.ahandle, pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .pTable = tsChildTableSdb}; sdbDeleteRow(&desc); + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); + } + + mnodeDestroySubMsg(pMsg); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + return code; } } @@ -1922,12 +1955,12 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) { - mError("msg:%p, app:%p table:%s, failed to alloc memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mError("msg:%p, app:%p table:%s, failed to alloc memory", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return TSDB_CODE_MND_OUT_OF_MEMORY; } pTable->info.type = (pCreate->numOfColumns == 0)? TSDB_CHILD_TABLE:TSDB_NORMAL_TABLE; - pTable->info.tableId = strdup(pCreate->tableId); + pTable->info.tableId = strdup(pCreate->tableName); pTable->createdTime = taosGetTimestampMs(); pTable->tid = tid; pTable->vgId = pVgroup->vgId; @@ -1943,7 +1976,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { size_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64); if (0 != strncasecmp(prefix, stableName, prefixLen)) { mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, stableName); + pCreate->tableName, stableName); mnodeDestroyChildTable(pTable); return TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; } @@ -1951,7 +1984,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(stableName); if (pMsg->pSTable == NULL) { mError("msg:%p, app:%p table:%s, corresponding super table:%s does not exist", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, stableName); + pCreate->tableName, stableName); mnodeDestroyChildTable(pTable); return TSDB_CODE_MND_INVALID_TABLE_NAME; } @@ -2013,12 +2046,12 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { .pMsg = pMsg, .fpReq = mnodeDoCreateChildTableFp }; - + int32_t code = sdbInsertRow(&desc); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeDestroyChildTable(pTable); pMsg->pTable = NULL; - mError("msg:%p, app:%p table:%s, failed to create, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, + mError("msg:%p, app:%p table:%s, failed to create, reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName, tstrerror(code)); } else { mDebug("msg:%p, app:%p table:%s, allocated in vgroup, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, @@ -2030,12 +2063,12 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { //SCMCreateTableMsg* p1 = pMsg->rpcMsg.pCont; // there are several tables here. - SCreateTableMsg* pCreate = (SCreateTableMsg*)(pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); + SCreateTableMsg* pCreate = (SCreateTableMsg*)((char *)pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg)); int32_t code = grantCheck(TSDB_GRANT_TIMESERIES); if (code != TSDB_CODE_SUCCESS) { mError("msg:%p, app:%p table:%s, failed to create, grant timeseries failed", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId); + pCreate->tableName); return code; } @@ -2046,7 +2079,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &tid); if (code != TSDB_CODE_SUCCESS) { mDebug("msg:%p, app:%p table:%s, failed to get available vgroup, reason:%s", pMsg, pMsg->rpcMsg.ahandle, - pCreate->tableId, tstrerror(code)); + pCreate->tableName, tstrerror(code)); return code; } @@ -2060,15 +2093,15 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { return mnodeDoCreateChildTable(pMsg, tid); } } else { - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableName); } if (pMsg->pTable == NULL) { - mError("msg:%p, app:%p table:%s, object not found, retry:%d reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pMsg->retry, + mError("msg:%p, app:%p table:%s, object not found, retry:%d reason:%s", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName, pMsg->retry, tstrerror(terrno)); return terrno; } else { - mDebug("msg:%p, app:%p table:%s, send create msg to vnode again", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId); + mDebug("msg:%p, app:%p table:%s, send create msg to vnode again", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableName); return mnodeDoCreateChildTableFp(pMsg); } } @@ -2084,7 +2117,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - tstrncpy(pDrop->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); + tstrncpy(pDrop->tableFname, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); pDrop->vgId = htonl(pTable->vgId); pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); pDrop->tid = htonl(pTable->tid); @@ -2093,7 +2126,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); mInfo("msg:%p, app:%p ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg, pMsg->rpcMsg.ahandle, - pDrop->tableId, pTable->vgId, pTable->tid, pTable->uid); + pDrop->tableFname, pTable->vgId, pTable->tid, pTable->uid); SRpcMsg rpcMsg = { .ahandle = pMsg, @@ -2115,7 +2148,7 @@ static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { SCTableObj *pTable = (SCTableObj *)pMsg->pTable; mError("msg:%p, app:%p ctable:%s, failed to drop, sdb error", pMsg, pMsg->rpcMsg.ahandle, pTable->info.tableId); return code; - } + } return mnodeSendDropChildTableMsg(pMsg, true); } @@ -2224,7 +2257,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 pTable->numOfColumns += ncols; pTable->sversion++; - + SAcctObj *pAcct = mnodeGetAcct(pDb->acct); if (pAcct != NULL) { pAcct->acctInfo.numOfTimeSeries += ncols; @@ -2287,7 +2320,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char } // int32_t rowSize = 0; - uint32_t len = strlen(newName); + uint32_t len = (uint32_t)strlen(newName); if (len >= TSDB_COL_NAME_LEN) { return TSDB_CODE_MND_COL_NAME_TOO_LONG; } @@ -2295,7 +2328,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char if (mnodeFindNormalTableColumnIndex(pTable, newName) >= 0) { return TSDB_CODE_MND_FIELD_ALREAY_EXIST; } - + // update SSchema *schema = (SSchema *) (pTable->schema + col); tstrncpy(schema->name, newName, sizeof(schema->name)); @@ -2335,7 +2368,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->tid = htonl(pTable->tid); pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; - tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); + tstrncpy(pMeta->tableFname, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); if (pTable->info.type == TSDB_CHILD_TABLE) { assert(pTable->superTable != NULL); @@ -2352,7 +2385,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->tversion = 0; pMeta->numOfTags = 0; pMeta->numOfColumns = htons((int16_t)pTable->numOfColumns); - pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromNormalTable(pMeta->schema, pTable); + pMeta->contLen = sizeof(STableMetaMsg) + mnodeSetSchemaFromNormalTable(pMeta->schema, pTable); } if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); @@ -2383,7 +2416,7 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { if (pMsg->rpcMsg.contLen <= sizeof(*pInfo)) { mError("msg:%p, app:%p table:%s, failed to auto create child table, tags not exist", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId); + pInfo->tableFname); return TSDB_CODE_MND_TAG_NOT_EXIST; } @@ -2398,7 +2431,7 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { int32_t totalLen = nameLen + tagLen + sizeof(int32_t)*2; if (tagLen == 0 || nameLen == 0) { mError("msg:%p, app:%p table:%s, failed to create table on demand for super table is empty, tagLen:%d", pMsg, - pMsg->rpcMsg.ahandle, pInfo->tableId, tagLen); + pMsg->rpcMsg.ahandle, pInfo->tableFname, tagLen); return TSDB_CODE_MND_INVALID_STABLE_NAME; } @@ -2406,15 +2439,14 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { SCMCreateTableMsg *pCreateMsg = calloc(1, contLen); if (pCreateMsg == NULL) { mError("msg:%p, app:%p table:%s, failed to create table while get meta info, no enough memory", pMsg, - pMsg->rpcMsg.ahandle, pInfo->tableId); + pMsg->rpcMsg.ahandle, pInfo->tableFname); return TSDB_CODE_MND_OUT_OF_MEMORY; } SCreateTableMsg* pCreate = (SCreateTableMsg*) ((char*) pCreateMsg + sizeof(SCMCreateTableMsg)); - size_t size = tListLen(pInfo->tableId); - tstrncpy(pCreate->tableId, pInfo->tableId, size); - tstrncpy(pCreate->db, pMsg->pDb->name, sizeof(pCreate->db)); + size_t size = tListLen(pInfo->tableFname); + tstrncpy(pCreate->tableName, pInfo->tableFname, size); pCreate->igExists = 1; pCreate->getMeta = 1; @@ -2427,7 +2459,7 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { memcpy(name, pInfo->tags + sizeof(int32_t), nameLen); mDebug("msg:%p, app:%p table:%s, start to create on demand, tagLen:%d stable:%s", pMsg, pMsg->rpcMsg.ahandle, - pInfo->tableId, tagLen, name); + pInfo->tableFname, tagLen, name); if (pMsg->rpcMsg.pCont != pMsg->pCont) { tfree(pMsg->rpcMsg.pCont); @@ -2491,7 +2523,7 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); mInfo("db:%s, all child tables will be dropped from sdb", pDropDb->name); @@ -2557,7 +2589,7 @@ static SCTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { return TSDB_CODE_COM_OPS_NOT_SUPPORT; -#if 0 +#if 0 SConfigTableMsg *pCfg = pMsg->rpcMsg.pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); pCfg->vgId = htonl(pCfg->vgId); @@ -2575,13 +2607,13 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { SMDCreateTableMsg *pCreate = NULL; pCreate = mnodeBuildCreateChildTableMsg(NULL, (SCTableObj *)pTable); mnodeDecTableRef(pTable); - + if (pCreate == NULL) return terrno; - + pMsg->rpcRsp.rsp = pCreate; pMsg->rpcRsp.len = htonl(pCreate->contLen); return TSDB_CODE_SUCCESS; -#endif +#endif } // handle drop child response @@ -2652,9 +2684,8 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (pMsg->pBatchMasterMsg) { ++pMsg->pBatchMasterMsg->successed; - if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received - >= pMsg->pBatchMasterMsg->expected) { - dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); } mnodeDestroySubMsg(pMsg); @@ -2674,7 +2705,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { .pMsg = pMsg, .fpRsp = mnodeDoCreateChildTableCb }; - + int32_t code = sdbInsertRowToQueue(&desc); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { pMsg->pTable = NULL; @@ -2682,6 +2713,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (pMsg->pBatchMasterMsg) { ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = code; if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, code); @@ -2720,6 +2752,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (pMsg->pBatchMasterMsg) { ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = rpcMsg->code; if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received >= pMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, rpcMsg->code); @@ -2782,7 +2815,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { SCTableObj *pTable = mnodeGetChildTable(tableId); if (pTable == NULL) continue; - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(tableId); if (pMsg->pDb == NULL || pMsg->pDb->status != TSDB_DB_STATUS_READY) { mnodeDecTableRef(pTable); continue; @@ -2821,7 +2854,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -2831,9 +2864,9 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void int32_t cols = 0; SSchema *pSchema = pMeta->schema; - SSchema s = tGetTableNameColumnSchema(); - pShow->bytes[cols] = s.bytes; - pSchema[cols].type = s.type; + SSchema* s = tGetTbnameColumnSchema(); + pShow->bytes[cols] = s->bytes; + pSchema[cols].type = s->type; strcpy(pSchema[cols].name, "table_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -2850,9 +2883,9 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - SSchema tbCol = tGetTableNameColumnSchema(); - pShow->bytes[cols] = tbCol.bytes + VARSTR_HEADER_SIZE; - pSchema[cols].type = tbCol.type; + SSchema* tbCol = tGetTbnameColumnSchema(); + pShow->bytes[cols] = tbCol->bytes + VARSTR_HEADER_SIZE; + pSchema[cols].type = tbCol->type; strcpy(pSchema[cols].name, "stable_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -2894,7 +2927,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return 0; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -2907,7 +2940,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; - int32_t prefixLen = tableIdPrefix(pDb->name, prefix, 64); + int32_t prefixLen = (int32_t)tableIdPrefix(pDb->name, prefix, 64); char* pattern = NULL; if (pShow->payloadLen > 0) { @@ -2931,7 +2964,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows } char tableName[TSDB_TABLE_NAME_LEN] = {0}; - + // pattern compare for table name mnodeExtractTableName(pTable->info.tableId, tableName); @@ -2960,13 +2993,13 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - + memset(tableName, 0, sizeof(tableName)); if (pTable->info.type == TSDB_CHILD_TABLE) { mnodeExtractTableName(pTable->superTable->info.tableId, tableName); STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tableName, pShow->bytes[cols]); } - + cols++; // uid @@ -3001,27 +3034,29 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { SAlterTableMsg *pAlter = pMsg->rpcMsg.pCont; mDebug("msg:%p, app:%p table:%s, alter table msg is received from thandle:%p", pMsg, pMsg->rpcMsg.ahandle, - pAlter->tableId, pMsg->rpcMsg.handle); + pAlter->tableFname, pMsg->rpcMsg.handle); - if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(pAlter->tableId); + if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableName(pAlter->tableFname); if (pMsg->pDb == NULL) { - mError("msg:%p, app:%p table:%s, failed to alter table, db not selected", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, db not selected", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); return TSDB_CODE_MND_DB_NOT_SELECTED; } - + if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); return TSDB_CODE_MND_DB_IN_DROPPING; } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { - mError("msg:%p, app:%p table:%s, failed to alter table, its log db", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, its log db", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif - if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableId); + if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pAlter->tableFname); if (pMsg->pTable == NULL) { - mError("msg:%p, app:%p table:%s, failed to alter table, table not exist", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mError("msg:%p, app:%p table:%s, failed to alter table, table not exist", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); return TSDB_CODE_MND_INVALID_TABLE_NAME; } @@ -3030,7 +3065,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { pAlter->tagValLen = htonl(pAlter->tagValLen); if (pAlter->numOfCols > 2) { - mError("msg:%p, app:%p table:%s, error numOfCols:%d in alter table", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId, + mError("msg:%p, app:%p table:%s, error numOfCols:%d in alter table", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname, pAlter->numOfCols); return TSDB_CODE_MND_APP_ERROR; } @@ -3041,7 +3076,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { int32_t code = TSDB_CODE_COM_OPS_NOT_SUPPORT; if (pMsg->pTable->type == TSDB_SUPER_TABLE) { - mDebug("msg:%p, app:%p table:%s, start to alter stable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mDebug("msg:%p, app:%p table:%s, start to alter stable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); if (pAlter->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { code = mnodeAddSuperTableTag(pMsg, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) { @@ -3057,7 +3092,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { } else { } } else { - mDebug("msg:%p, app:%p table:%s, start to alter ctable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableId); + mDebug("msg:%p, app:%p table:%s, start to alter ctable", pMsg, pMsg->rpcMsg.ahandle, pAlter->tableFname); if (pAlter->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) { return TSDB_CODE_COM_OPS_NOT_SUPPORT; } else if (pAlter->type == TSDB_ALTER_TABLE_ADD_COLUMN) { @@ -3076,7 +3111,7 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return TSDB_CODE_MND_DB_NOT_SELECTED; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); @@ -3086,9 +3121,9 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo int32_t cols = 0; SSchema *pSchema = pMeta->schema; - SSchema tbnameColSchema = tGetTableNameColumnSchema(); - pShow->bytes[cols] = tbnameColSchema.bytes; - pSchema[cols].type = tbnameColSchema.type; + SSchema* tbnameColSchema = tGetTbnameColumnSchema(); + pShow->bytes[cols] = tbnameColSchema->bytes; + pSchema[cols].type = tbnameColSchema->type; strcpy(pSchema[cols].name, "table_name"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -3129,13 +3164,13 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t rows, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); if (pDb == NULL) return 0; - + if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); mnodeDecDbRef(pDb); return 0; } - + int32_t numOfRows = 0; SCTableObj *pTable = NULL; SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; @@ -3143,12 +3178,12 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro char prefix[64] = {0}; tstrncpy(prefix, pDb->name, 64); strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = (int32_t)strlen(prefix); while (numOfRows < rows) { pShow->pIter = mnodeGetNextChildTable(pShow->pIter, &pTable); if (pTable == NULL) break; - + // not belong to current db if (strncmp(pTable->info.tableId, prefix, prefixLen) || pTable->info.type != TSDB_STREAM_TABLE) { mnodeDecTableRef(pTable); @@ -3156,7 +3191,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro } char tableName[TSDB_TABLE_NAME_LEN] = {0}; - + // pattern compare for table name mnodeExtractTableName(pTable->info.tableId, tableName); diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index fb26086d0433d7760c9cc7ed906abb570a4fcfee..55ee39b6bceb7a3d5ebc8838e1a9a2151ccb6c3d 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -123,12 +123,12 @@ static void mnodePrintUserAuth() { mnodeDecUserRef(pUser); } - fflush(fp); + fsync(fileno(fp)); fclose(fp); } static int32_t mnodeUserActionRestored() { - int32_t numOfRows = sdbGetNumOfRows(tsUserSdb); + int64_t numOfRows = sdbGetNumOfRows(tsUserSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { mInfo("dnode first deploy, create root user"); SAcctObj *pAcct = mnodeGetAcct(TSDB_DEFAULT_USER); @@ -148,14 +148,14 @@ static int32_t mnodeUserActionRestored() { int32_t mnodeInitUsers() { SUserObj tObj; - tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsUserUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_USER, .name = "users", .hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE, .maxRowSize = tsUserUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_STRING, .fpInsert = mnodeUserActionInsert, .fpDelete = mnodeUserActionDelete, @@ -204,11 +204,11 @@ void mnodeCancelGetNextUser(void *pIter) { } void mnodeIncUserRef(SUserObj *pUser) { - return sdbIncRef(tsUserSdb, pUser); + sdbIncRef(tsUserSdb, pUser); } void mnodeDecUserRef(SUserObj *pUser) { - return sdbDecRef(tsUserSdb, pUser); + sdbDecRef(tsUserSdb, pUser); } static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) { @@ -337,7 +337,7 @@ static int32_t mnodeGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pCo cols++; pMeta->numOfColumns = htons(cols); - strcpy(pMeta->tableId, "show users"); + strcpy(pMeta->tableFname, "show users"); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -561,7 +561,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) { void mnodeDropAllUsers(SAcctObj *pAcct) { void * pIter = NULL; int32_t numOfUsers = 0; - int32_t acctNameLen = strlen(pAcct->user); + int32_t acctNameLen = (int32_t)strlen(pAcct->user); SUserObj *pUser = NULL; while (1) { diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 881ef95fca0ac327d75ab00007b1bab3c8980578..7eb3122d83032ed9759236bf999210008cdb908c 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -60,6 +60,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg); static void mnodeProcessAlterVnodeRsp(SRpcMsg *rpcMsg); +static void mnodeProcessSyncVnodeRsp(SRpcMsg *rpcMsg); static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) ; static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle); @@ -206,14 +207,14 @@ static int32_t mnodeVgroupActionRestored() { int32_t mnodeInitVgroups() { SVgObj tObj; - tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + tsVgUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj); SSdbTableDesc desc = { .id = SDB_TABLE_VGROUP, .name = "vgroups", .hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE, .maxRowSize = tsVgUpdateSize, - .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj), .keyType = SDB_KEY_AUTO, .fpInsert = mnodeVgroupActionInsert, .fpDelete = mnodeVgroupActionDelete, @@ -236,6 +237,7 @@ int32_t mnodeInitVgroups() { mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_VGROUP, mnodeCancelGetNextVgroup); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp); + mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessSyncVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp); mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_CONFIG_VNODE, mnodeProcessVnodeCfgMsg); @@ -245,11 +247,11 @@ int32_t mnodeInitVgroups() { } void mnodeIncVgroupRef(SVgObj *pVgroup) { - return sdbIncRef(tsVgroupSdb, pVgroup); + sdbIncRef(tsVgroupSdb, pVgroup); } void mnodeDecVgroupRef(SVgObj *pVgroup) { - return sdbDecRef(tsVgroupSdb, pVgroup); + sdbDecRef(tsVgroupSdb, pVgroup); } SVgObj *mnodeGetVgroup(int32_t vgId) { @@ -367,6 +369,11 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) { maxIdPoolSize = MAX(maxIdPoolSize, idPoolSize); } + // create one table each vnode + if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) { + maxIdPoolSize = 1; + } + // new vgroup if (pInputVgroup->idPool == NULL) { pInputVgroup->idPool = taosInitIdPool(maxIdPoolSize); @@ -379,6 +386,11 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) { } } + // create one table each vnode + if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) { + return TSDB_CODE_SUCCESS; + } + // realloc all vgroups in db int32_t newIdPoolSize; if (minIdPoolSize * 4 < tsTableIncStepPerVnode) { @@ -433,6 +445,7 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi mDebug("msg:%p, app:%p db:%s, no enough sid in vgId:%d", pMsg, pMsg->rpcMsg.ahandle, pDb->name, pVgroup->vgId); continue; } + mTrace("vgId:%d, alloc tid:%d", pVgroup->vgId, sid); *pSid = sid; *ppVgroup = pVgroup; @@ -449,6 +462,10 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi maxVgroupsPerDb = MIN(maxVgroupsPerDb, TSDB_MAX_VNODES_PER_DB); } + if (pDb->cfg.dbType == TSDB_DB_TYPE_TOPIC) { + maxVgroupsPerDb = TSDB_MAX_DB_PARTITON_OPTION; + } + int32_t code = TSDB_CODE_MND_NO_ENOUGH_DNODES; if (pDb->numOfVgroups < maxVgroupsPerDb) { mDebug("msg:%p, app:%p db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg, @@ -493,6 +510,7 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi pDb->vgListIndex = 0; pthread_mutex_unlock(&pDb->mutex); + mTrace("vgId:%d, alloc tid:%d", pVgroup->vgId, sid); return TSDB_CODE_SUCCESS; } @@ -534,6 +552,20 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { tstrerror(code)); SSdbRow desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .pTable = tsVgroupSdb}; sdbDeleteRow(&desc); + + if (pMsg->pBatchMasterMsg) { + ++pMsg->pBatchMasterMsg->received; + pMsg->pBatchMasterMsg->code = pMsg->code; + if (pMsg->pBatchMasterMsg->successed + pMsg->pBatchMasterMsg->received + >= pMsg->pBatchMasterMsg->expected) { + dnodeSendRpcMWriteRsp(pMsg->pBatchMasterMsg, pMsg->code); + } + + mnodeDestroySubMsg(pMsg); + + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + return code; } else { mInfo("msg:%p, app:%p vgId:%d, is created in sdb, db:%s replica:%d", pMsg, pMsg->rpcMsg.ahandle, pVgroup->vgId, @@ -804,26 +836,37 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v return numOfRows; } -void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable) { +int32_t mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable, bool needCheck) { int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool); if (pTable->tid > idPoolSize) { mnodeAllocVgroupIdPool(pVgroup); } if (pTable->tid >= 1) { - taosIdPoolMarkStatus(pVgroup->idPool, pTable->tid); - pVgroup->numOfTables++; - // The create vgroup message may be received later than the create table message - // and the writing order in sdb is therefore uncertain - // which will cause the reference count of the vgroup to be incorrect when restarting - // mnodeIncVgroupRef(pVgroup); + if (taosIdPoolMarkStatus(pVgroup->idPool, pTable->tid) || !needCheck) { + pVgroup->numOfTables++; + mTrace("table:%s, vgId:%d tid:%d, mark tid used, uid:%" PRIu64, pTable->info.tableId, pTable->vgId, pTable->tid, + pTable->uid); + // The create vgroup message may be received later than the create table message + // and the writing order in sdb is therefore uncertain + // which will cause the reference count of the vgroup to be incorrect when restarting + // mnodeIncVgroupRef(pVgroup); + } else { + mError("table:%s, vgId:%d tid:%d, failed to mark tid, uid:%" PRIu64, pTable->info.tableId, pTable->vgId, + pTable->tid, pTable->uid); + return -1; + } } + + return 0; } void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable) { if (pTable->tid >= 1) { taosFreeId(pVgroup->idPool, pTable->tid); pVgroup->numOfTables--; + mTrace("table:%s, vgId:%d tid:%d, put tid back uid:%" PRIu64, pTable->info.tableId, pTable->vgId, pTable->tid, + pTable->uid); // The create vgroup message may be received later than the create table message // and the writing order in sdb is therefore uncertain // which will cause the reference count of the vgroup to be incorrect when restarting @@ -867,6 +910,7 @@ static SCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { pCfg->update = pDb->cfg.update; pCfg->cacheLastRow = pDb->cfg.cacheLastRow; pCfg->dbReplica = pDb->cfg.replications; + pCfg->dbType = pDb->cfg.dbType; SVnodeDesc *pNodes = pVnode->nodes; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { @@ -925,6 +969,38 @@ void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) { } } +static SSyncVnodeMsg *mnodeBuildSyncVnodeMsg(int32_t vgId) { + SSyncVnodeMsg *pSyncVnode = rpcMallocCont(sizeof(SSyncVnodeMsg)); + if (pSyncVnode == NULL) return NULL; + + pSyncVnode->vgId = htonl(vgId); + return pSyncVnode; +} + +static void mnodeSendSyncVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) { + SSyncVnodeMsg *pSyncVnode = mnodeBuildSyncVnodeMsg(pVgroup->vgId); + SRpcMsg rpcMsg = { + .ahandle = NULL, + .pCont = pSyncVnode, + .contLen = pSyncVnode ? sizeof(SSyncVnodeMsg) : 0, + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_SYNC_VNODE + }; + + dnodeSendMsgToDnode(epSet, &rpcMsg); +} + +void mnodeSendSyncVgroupMsg(SVgObj *pVgroup) { + mDebug("vgId:%d, send sync all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, + pVgroup->dbName); + for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { + SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + mDebug("vgId:%d, index:%d, send sync vnode msg to dnode %s", pVgroup->vgId, i, + pVgroup->vnodeGid[i].pDnode->dnodeEp); + mnodeSendSyncVnodeMsg(pVgroup, &epSet); + } +} + static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) { SCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { @@ -952,6 +1028,10 @@ static void mnodeProcessAlterVnodeRsp(SRpcMsg *rpcMsg) { mDebug("alter vnode rsp received"); } +static void mnodeProcessSyncVnodeRsp(SRpcMsg *rpcMsg) { + mDebug("sync vnode rsp received"); +} + static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (rpcMsg->ahandle == NULL) return; @@ -989,6 +1069,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->pBatchMasterMsg) { ++mnodeMsg->pBatchMasterMsg->received; + mnodeMsg->pBatchMasterMsg->code = code; if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received >= mnodeMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, code); @@ -1011,6 +1092,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->pBatchMasterMsg) { ++mnodeMsg->pBatchMasterMsg->received; + mnodeMsg->pBatchMasterMsg->code = mnodeMsg->code; if (mnodeMsg->pBatchMasterMsg->successed + mnodeMsg->pBatchMasterMsg->received >= mnodeMsg->pBatchMasterMsg->expected) { dnodeSendRpcMWriteRsp(mnodeMsg->pBatchMasterMsg, mnodeMsg->code); diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index f41719f2404297070656910653b65a3fbffa7916..ab8b0f76785c24e3385f49245f6e191b2d57cc40 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_LINUX) diff --git a/src/os/inc/eok.h b/src/os/inc/eok.h new file mode 100644 index 0000000000000000000000000000000000000000..0874ca975b44a2e355c7eb746af4f30b4d163610 --- /dev/null +++ b/src/os/inc/eok.h @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ +#define _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __APPLE__ + +enum EPOLL_EVENTS + { + EPOLLIN = 0x001, +#define EPOLLIN EPOLLIN + EPOLLPRI = 0x002, +#define EPOLLPRI EPOLLPRI + EPOLLOUT = 0x004, +#define EPOLLOUT EPOLLOUT + EPOLLRDNORM = 0x040, +#define EPOLLRDNORM EPOLLRDNORM + EPOLLRDBAND = 0x080, +#define EPOLLRDBAND EPOLLRDBAND + EPOLLWRNORM = 0x100, +#define EPOLLWRNORM EPOLLWRNORM + EPOLLWRBAND = 0x200, +#define EPOLLWRBAND EPOLLWRBAND + EPOLLMSG = 0x400, +#define EPOLLMSG EPOLLMSG + EPOLLERR = 0x008, +#define EPOLLERR EPOLLERR + EPOLLHUP = 0x010, +#define EPOLLHUP EPOLLHUP + EPOLLRDHUP = 0x2000, +#define EPOLLRDHUP EPOLLRDHUP + EPOLLEXCLUSIVE = 1u << 28, +#define EPOLLEXCLUSIVE EPOLLEXCLUSIVE + EPOLLWAKEUP = 1u << 29, +#define EPOLLWAKEUP EPOLLWAKEUP + EPOLLONESHOT = 1u << 30, +#define EPOLLONESHOT EPOLLONESHOT + EPOLLET = 1u << 31 +#define EPOLLET EPOLLET + }; + +/* Valid opcodes ( "op" parameter ) to issue to epoll_ctl(). */ +#define EPOLL_CTL_ADD 1 /* Add a file descriptor to the interface. */ +#define EPOLL_CTL_DEL 2 /* Remove a file descriptor from the interface. */ +#define EPOLL_CTL_MOD 3 /* Change file descriptor epoll_event structure. */ + + +typedef union epoll_data +{ + void *ptr; + int fd; + uint32_t u32; + uint64_t u64; +} epoll_data_t; + +struct epoll_event +{ + uint32_t events; /* Epoll events */ + epoll_data_t data; /* User data variable */ +}; + +int epoll_create(int size); +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event); +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout); +int epoll_close(int epfd); + +#endif // __APPLE__ + +#ifdef __cplusplus +} +#endif + +#endif // _eok_h_fd274616_996c_400e_9023_ae70be881fa3_ + diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 9383ae48dc5c6151aea0d9e8a2641603b63da144..8312b74a5015aeb0991b303f0ea8f207cb2cf3eb 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -62,6 +62,7 @@ extern "C" { #include "osMemory.h" #include "osRand.h" #include "osSemphone.h" +#include "osSignal.h" #include "osSocket.h" #include "osString.h" #include "osSysinfo.h" diff --git a/src/os/inc/osArm32.h b/src/os/inc/osArm32.h index 24ff95522ebc7ca2b2c23e9961371df58aa2fc6d..54835a1ca8d95b1b05707f165301ac5defc3e961 100644 --- a/src/os/inc/osArm32.h +++ b/src/os/inc/osArm32.h @@ -77,6 +77,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzll(val) diff --git a/src/os/inc/osArm64.h b/src/os/inc/osArm64.h index 22f0000e96c5083556a3751e57f5bdb18d686332..76098f684617abb7d96a17c2a13238b82cb69cef 100644 --- a/src/os/inc/osArm64.h +++ b/src/os/inc/osArm64.h @@ -78,6 +78,7 @@ extern "C" { #include #include #include +#include #ifdef __cplusplus } diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 1461ec6d3b177c300e667d93043308d93ea67d0f..7c206afe7a12e9be17ad997ee50625a0cce438e9 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -75,23 +75,29 @@ extern "C" { #define TAOS_OS_FUNC_FILE_SENDIFLE #define TAOS_OS_FUNC_SEMPHONE - #define tsem_t dispatch_semaphore_t - int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value); - int tsem_wait(dispatch_semaphore_t *sem); - int tsem_post(dispatch_semaphore_t *sem); - int tsem_destroy(dispatch_semaphore_t *sem); + typedef struct tsem_s *tsem_t; + int tsem_init(tsem_t *sem, int pshared, unsigned int value); + int tsem_wait(tsem_t *sem); + int tsem_post(tsem_t *sem); + int tsem_destroy(tsem_t *sem); #define TAOS_OS_FUNC_SOCKET_SETSOCKETOPT #define TAOS_OS_FUNC_STRING_STR2INT64 #define TAOS_OS_FUNC_SYSINFO #define TAOS_OS_FUNC_TIMER +#define TAOS_OS_FUNC_SEMPHONE_PTHREAD // specific #define htobe64 htonll typedef int(*__compar_fn_t)(const void *, const void *); // for send function in tsocket.c +#if defined(MSG_NOSIGNAL) +#undef MSG_NOSIGNAL +#endif + #define MSG_NOSIGNAL 0 + #define SO_NO_CHECK 0x1234 #define SOL_TCP 0x1234 #define TCP_KEEPIDLE 0x1234 @@ -100,6 +106,20 @@ typedef int(*__compar_fn_t)(const void *, const void *); #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE #endif +#define TAOS_OS_FUNC_PTHREAD_RWLOCK + +int64_t tsosStr2int64(char *str); + +#include "eok.h" + +void taos_block_sigalrm(void); + +#define TAOS_OS_DEF_EPOLL + #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h index d718bef6da42f59c5ee4ea7bfaad098b47062984..bbe0f98ec0b632deb39691f74350bced6d2a6515 100644 --- a/src/os/inc/osDef.h +++ b/src/os/inc/osDef.h @@ -20,8 +20,10 @@ extern "C" { #endif -#ifndef STDERR_FILENO -#define STDERR_FILENO (2) +#ifndef WINDOWS + #ifndef O_BINARY + #define O_BINARY 0 + #endif #endif #define FD_VALID(x) ((x) > STDERR_FILENO) @@ -81,6 +83,20 @@ extern "C" { } \ } while (0) +#define DEFAULT_DOUBLE_COMP(x, y) \ + do { \ + if (isnan(x) && isnan(y)) { return 0; } \ + if (isnan(x)) { return -1; } \ + if (isnan(y)) { return 1; } \ + if ((x) == (y)) { \ + return 0; \ + } else { \ + return (x) < (y) ? -1 : 1; \ + } \ + } while (0) + +#define DEFAULT_FLOAT_COMP(x, y) DEFAULT_DOUBLE_COMP(x, y) + #define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) // align to 8bytes @@ -90,11 +106,11 @@ extern "C" { #ifdef _ISOC11_SOURCE #define threadlocal _Thread_local #elif defined(__APPLE__) - #define threadlocal + #define threadlocal __thread #elif defined(__GNUC__) && !defined(threadlocal) #define threadlocal __thread #else - #define threadlocal + #define threadlocal __declspec( thread ) #endif #ifdef __cplusplus diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h index 67cfdb3b530a7db17b834a1443d681e1cd5361a7..540efc310c1d89530d6e41733ce2419ad79b1e36 100644 --- a/src/os/inc/osDir.h +++ b/src/os/inc/osDir.h @@ -25,8 +25,8 @@ extern "C" { // TAOS_OS_FUNC_DIR void taosRemoveDir(char *rootDir); int taosMkDir(const char *pathname, mode_t mode); -void taosRename(char* oldName, char *newName); void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays); +int32_t taosRename(char* oldName, char *newName); int32_t taosCompressFile(char *srcFileName, char *destFileName); #ifdef __cplusplus diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index c9b3b9cd76d675169746a2478bfbd775d6da190a..c117ae4039d05b6a063726ec9052fe5f5562365e 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -20,10 +20,13 @@ extern "C" { #endif +#include "osSocket.h" + int64_t taosReadImp(int32_t fd, void *buf, int64_t count); int64_t taosWriteImp(int32_t fd, void *buf, int64_t count); int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence); int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath); +int64_t taosCopy(char *from, char *to); #define taosRead(fd, buf, count) taosReadImp(fd, buf, count) #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count) @@ -37,7 +40,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP } // TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size); int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); #ifdef TAOS_RANDOM_FILE_FAIL diff --git a/src/os/inc/osLinux32.h b/src/os/inc/osLinux32.h index cfef05368fe76f14604a91f1551b850a79898a4b..eee4999399ace745e6800f81cb09144e828f8cfc 100644 --- a/src/os/inc/osLinux32.h +++ b/src/os/inc/osLinux32.h @@ -77,6 +77,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzll(val) diff --git a/src/os/inc/osLinux64.h b/src/os/inc/osLinux64.h index a2febd51b7b769439b681aaf1871fe514b5558f3..0dfc819da339bfba77b5778aa27a8b60c48ca447 100644 --- a/src/os/inc/osLinux64.h +++ b/src/os/inc/osLinux64.h @@ -75,6 +75,7 @@ extern "C" { #include #include #include +#include #ifndef _ALPINE #include #endif diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h index 439e4cab72e4192d9c13a02519d268586a06f13e..2cf7e14d2f4bc9fc124cdf4de167c5b2cb93f4bb 100644 --- a/src/os/inc/osMemory.h +++ b/src/os/inc/osMemory.h @@ -35,7 +35,7 @@ void taosDumpMemoryLeak(); void * taosTMalloc(size_t size); void * taosTCalloc(size_t nmemb, size_t size); void * taosTRealloc(void *ptr, size_t size); -void taosTZfree(void *ptr); +void * taosTZfree(void *ptr); size_t taosTSizeof(void *ptr); void taosTMemset(void *ptr, int c); diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index 74e1bd487815942651111a2aa85e31650281bf20..3332a9234b040aaa49c1d097e63f03a6c9bde25b 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -28,6 +28,21 @@ extern "C" { #define tsem_destroy sem_destroy #endif +#ifdef TAOS_OS_FUNC_PTHREAD_RWLOCK + #define pthread_rwlock_t pthread_mutex_t + #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL) + #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock) + #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock) + #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock) + #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock) + + #define pthread_spinlock_t pthread_mutex_t + #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL) + #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock) + #define pthread_spin_lock(lock) pthread_mutex_lock(lock) + #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock) +#endif + // TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread); int64_t taosGetSelfPthreadId(); diff --git a/src/dnode/inc/dnodeMgmt.h b/src/os/inc/osSignal.h similarity index 50% rename from src/dnode/inc/dnodeMgmt.h rename to src/os/inc/osSignal.h index 2038ef5286b32522b11409ba5a253b33228b984d..c13cd83178691fa8c617369904cb37a20ab27479 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/os/inc/osSignal.h @@ -13,35 +13,48 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_DNODE_MGMT_H -#define TDENGINE_DNODE_MGMT_H +#ifndef TDENGINE_OS_SIGNAL_H +#define TDENGINE_OS_SIGNAL_H #ifdef __cplusplus extern "C" { #endif -#include "trpc.h" +#include "os.h" +#include "taosdef.h" +#include -int32_t dnodeInitMgmt(); -void dnodeCleanupMgmt(); -int32_t dnodeInitMgmtTimer(); -void dnodeCleanupMgmtTimer(); -void dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg); +#ifndef SIGALRM + #define SIGALRM 1234 +#endif + +#ifndef SIGHUP + #define SIGHUP 1230 +#endif -void* dnodeGetVnode(int32_t vgId); -int32_t dnodeGetVnodeStatus(void *pVnode); -void* dnodeGetVnodeRworker(void *pVnode); -void* dnodeGetVnodeWworker(void *pVnode); -void* dnodeGetVnodeWal(void *pVnode); -void* dnodeGetVnodeTsdb(void *pVnode); -void dnodeReleaseVnode(void *pVnode); +#ifndef SIGCHLD + #define SIGCHLD 1234 +#endif -void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); -void dnodeGetEpSetForPeer(SRpcEpSet *epSet); -void dnodeGetEpSetForShell(SRpcEpSet *epSet); +#ifndef SIGUSR1 + #define SIGUSR1 1234 +#endif + +#ifndef SIGUSR2 + #define SIGUSR2 1234 +#endif + +#ifndef SIGBREAK + #define SIGBREAK 1234 +#endif + +typedef void (*FSignalHandler)(int32_t signum, void *sigInfo, void *context); +void taosSetSignal(int32_t signum, FSignalHandler sigfp); +void taosIgnSignal(int32_t signum); +void taosDflSignal(int32_t signum); #ifdef __cplusplus } #endif -#endif +#endif // TDENGINE_TTIME_H diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 13d3fa407921ebf93992d3d0da5c8dabc41f5e44..cf75b74a9afa75caa15ec82e191d781159af944f 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -33,11 +33,13 @@ extern "C" { x = FD_INITIALIZER; \ } \ } - typedef int32_t SOCKET; #endif #ifndef TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 500 + typedef int32_t SOCKET; + typedef SOCKET EpollFd; + #define EpollClose(pollFd) taosCloseSocket(pollFd) #endif #ifdef TAOS_RANDOM_NETWORK_FAIL @@ -61,10 +63,13 @@ extern "C" { int32_t taosSetNonblocking(SOCKET sock, int32_t on); void taosIgnSIGPIPE(); void taosBlockSIGPIPE(); +void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); +int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t* optlen); + // TAOS_OS_FUNC_SOCKET_INET uint32_t taosInetAddr(char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); diff --git a/src/os/inc/osSysinfo.h b/src/os/inc/osSysinfo.h index b592a6c679b41ecc2a30e93cfd9d1250da647bd5..895b5dd499196703f954c5621241cd2d1e2fabec 100644 --- a/src/os/inc/osSysinfo.h +++ b/src/os/inc/osSysinfo.h @@ -21,10 +21,17 @@ extern "C" { #endif // TAOS_OS_FUNC_SYSINFO +typedef struct { + int64_t tsize; + int64_t used; + int64_t avail; +} SysDiskSize; + +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize); void taosGetSystemInfo(); bool taosGetProcIO(float *readKB, float *writeKB); bool taosGetBandSpeed(float *bandSpeedKb); -bool taosGetDisk(); +void taosGetDisk(); bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ; bool taosGetProcMemory(float *memoryUsedMB) ; bool taosGetSysMemory(float *memoryUsedMB); diff --git a/src/os/inc/osTime.h b/src/os/inc/osTime.h index b20ccadadb22a04733d97bd19b919660ee677d0d..2c50e7eeabd934a9d88f32d4b3bdc29d41d699b1 100644 --- a/src/os/inc/osTime.h +++ b/src/os/inc/osTime.h @@ -72,6 +72,11 @@ typedef struct SInterval { int64_t offset; } SInterval; +typedef struct SSessionWindow { + int64_t gap; // gap between two session window(in microseconds) + int32_t primaryColId; // primary timestamp column +} SSessionWindow; + int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision); int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision); int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision); diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 5a1e6425728b88a5461ce3fe83a060e23161ae86..a058f2cc99029e311168428ce6e91f88212a4c73 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -46,6 +46,8 @@ #include "msvcFcntl.h" #include "msvcLibgen.h" #include "msvcStdio.h" +#include "msvcUnistd.h" +#include "msvcLibgen.h" #include "sys/msvcStat.h" #include "sys/msvcTypes.h" @@ -67,6 +69,8 @@ extern "C" { #define TAOS_OS_FUNC_FILE_GETTMPFILEPATH #define TAOS_OS_FUNC_FILE_FTRUNCATE +#define TAOS_OS_FUNC_DIR + #define TAOS_OS_FUNC_MATH #define SWAP(a, b, c) \ do { \ @@ -93,6 +97,12 @@ typedef SOCKET eventfd_t; #define TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 100 + typedef SOCKET EpollFd; + #define EpollClose(pollFd) epoll_close(pollFd) + +#ifndef EPOLLWAKEUP + #define EPOLLWAKEUP (1u << 29) +#endif #define TAOS_OS_DEF_ZU #define PRIzu "ld" @@ -138,7 +148,6 @@ typedef int (*__compar_fn_t)(const void *, const void *); #define in_addr_t unsigned long #define socklen_t int #define htobe64 htonll -#define getpid _getpid struct tm *localtime_r(const time_t *timep, struct tm *result); char * strptime(const char *buf, const char *fmt, struct tm *tm); @@ -147,15 +156,8 @@ char * getpass(const char *prefix); int flock(int fd, int option); int fsync(int filedes); char * strndup(const char *s, size_t n); -char * dirname(char *pszPathname); int gettimeofday(struct timeval *ptv, void *pTimeZone); -// for access function in io.h -#define F_OK 00 //Existence only -#define W_OK 02 //Write - only -#define R_OK 04 //Read - only -#define X_OK 06 //Read and write - // for send function in tsocket.c #define MSG_NOSIGNAL 0 #define SO_NO_CHECK 0x1234 @@ -191,23 +193,23 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); #define PATH_MAX 256 #endif -//for signal, not dispose -#define SIGALRM 1234 -typedef int sigset_t; -struct sigaction { - void (*sa_handler)(int); -}; -int sigaction(int, struct sigaction *, void *); +#define TAOS_OS_FUNC_SIGNAL typedef struct { int we_wordc; - char **we_wordv; + char *we_wordv[1]; int we_offs; - char wordPos[20]; + char wordPos[1025]; } wordexp_t; -int wordexp(const char *words, wordexp_t *pwordexp, int flags); +int wordexp(char *words, wordexp_t *pwordexp, int flags); void wordfree(wordexp_t *pwordexp); +#define openlog(a, b, c) +#define closelog() +#define LOG_ERR 0 +#define LOG_INFO 1 +void syslog(int unused, const char *format, ...); + #define TAOS_OS_FUNC_ATOMIC #define atomic_load_8(ptr) (*(char volatile*)(ptr)) #define atomic_load_16(ptr) (*(short volatile*)(ptr)) diff --git a/src/os/src/alpine/CMakeLists.txt b/src/os/src/alpine/CMakeLists.txt index daa0b3cf43d0de60fafc960e48e3ad8aeec1a9ad..b5e739c24ce7ec3ef3ffc537ca8769706f7b56de 100644 --- a/src/os/src/alpine/CMakeLists.txt +++ b/src/os/src/alpine/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) diff --git a/src/os/src/darwin/CMakeLists.txt b/src/os/src/darwin/CMakeLists.txt index 7f05ddd64b87f69e5fa03c874bb2bc401e5094cc..c4cb28aa05e4716ca98c2687ce41d436b1300bb2 100644 --- a/src/os/src/darwin/CMakeLists.txt +++ b/src/os/src/darwin/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 6adebabec01ae378a4c5f3d82389508334210972..8ac97eacfbc53eb67fa141f8bdddc79d8a65777c 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -17,16 +17,27 @@ #include "os.h" #include "tglobal.h" +static const char* expand_like_shell(const char *path) { + static __thread char buf[TSDB_FILENAME_LEN]; + buf[0] = '\0'; + wordexp_t we; + if (wordexp(path, &we, 0)) return "/tmp/taosd"; + if (sizeof(buf)<=snprintf(buf, sizeof(buf), "%s", we.we_wordv[0])) return "/tmp/taosd"; + wordfree(&we); + return buf; +} + void osInit() { if (configDir[0] == 0) { - strcpy(configDir, "~/TDengine/cfg"); + strcpy(configDir, expand_like_shell("/usr/local/etc/taos")); } + strcpy(tsDataDir, expand_like_shell("/usr/local/var/lib/taos")); + strcpy(tsLogDir, expand_like_shell("/usr/local/var/log/taos")); + strcpy(tsScriptDir, expand_like_shell("/usr/local/etc/taos")); strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); - strcpy(tsDataDir, "~/TDengine/data"); - strcpy(tsLogDir, "~/TDengine/log"); - strcpy(tsScriptDir, "~/TDengine/cfg"); strcpy(tsOsName, "Darwin"); } + diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c index dacf4db74137b8cc58db2b390ba971cf9dfbc012..4236ea1c5f08c8b8a8a81ae10c7b442ad3b5ce6a 100644 --- a/src/os/src/darwin/darwinFile.c +++ b/src/os/src/darwin/darwinFile.c @@ -17,41 +17,49 @@ #include "os.h" #include "tulog.h" -#define _SEND_FILE_STEP_ 1000 - int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { - fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = {0}; - - for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); - if (rlen <= 0) { - return writeLen; - } else if (rlen < _SEND_FILE_STEP_) { - fwrite(buffer, 1, rlen, out_file); - return (int)(writeLen + rlen); - } else { - fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); - writeLen += _SEND_FILE_STEP_; - } + int r = 0; + if (offset) { + r = fseek(in_file, *offset, SEEK_SET); + if (r==-1) return -1; } - - int remain = count - writeLen; - if (remain > 0) { - size_t rlen = fread(buffer, 1, remain, in_file); - if (rlen <= 0) { - return writeLen; - } else { - fwrite(buffer, 1, remain, out_file); - writeLen += remain; + off_t len = count; + while (len>0) { + char buf[1024*16]; + off_t n = sizeof(buf); + if (len0) { + char buf[1024*16]; + off_t n = sizeof(buf); + if (len. */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #define _DEFAULT_SOURCE #include "os.h" -int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value) { - *sem = dispatch_semaphore_create(value); - if (*sem == NULL) { +#include + +// #define SEM_USE_PTHREAD +// #define SEM_USE_POSIX +#define SEM_USE_SEM + +#ifdef SEM_USE_SEM +#include +#include +#include +#include +#include + +static pthread_t sem_thread; +static pthread_once_t sem_once; +static task_t sem_port; +static volatile int sem_inited = 0; +static semaphore_t sem_exit; + +static void* sem_thread_routine(void *arg) { + (void)arg; + sem_port = mach_task_self(); + kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0); + if (ret != KERN_SUCCESS) { + fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", basename(__FILE__), __LINE__, __func__); + sem_inited = -1; + return NULL; + } + sem_inited = 1; + semaphore_wait(sem_exit); + return NULL; +} + +static void once_init(void) { + int r = 0; + r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__); + return; + } + while (sem_inited==0) { + ; + } +} +#endif + +struct tsem_s { +#ifdef SEM_USE_PTHREAD + pthread_mutex_t lock; + pthread_cond_t cond; + volatile int64_t val; +#elif defined(SEM_USE_POSIX) + size_t id; + sem_t *sem; +#elif defined(SEM_USE_SEM) + semaphore_t sem; +#else // SEM_USE_PTHREAD + dispatch_semaphore_t sem; +#endif // SEM_USE_PTHREAD + + volatile unsigned int valid:1; +}; + +int tsem_init(tsem_t *sem, int pshared, unsigned int value) { + // fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem); + if (*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = (struct tsem_s*)calloc(1, sizeof(*p)); + if (!p) { + fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + +#ifdef SEM_USE_PTHREAD + int r = pthread_mutex_init(&p->lock, NULL); + do { + if (r) break; + r = pthread_cond_init(&p->cond, NULL); + if (r) { + pthread_mutex_destroy(&p->lock); + break; + } + p->val = value; + } while (0); + if (r) { + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#elif defined(SEM_USE_POSIX) + static size_t tick = 0; + do { + size_t id = atomic_add_fetch_64(&tick, 1); + if (id==SEM_VALUE_MAX) { + atomic_store_64(&tick, 0); + id = 0; + } + char name[NAME_MAX-4]; + snprintf(name, sizeof(name), "/t%ld", id); + p->sem = sem_open(name, O_CREAT|O_EXCL, pshared, value); + p->id = id; + if (p->sem!=SEM_FAILED) break; + int e = errno; + if (e==EEXIST) continue; + if (e==EINTR) continue; + fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + abort(); + } while (p->sem==SEM_FAILED); +#elif defined(SEM_USE_SEM) + pthread_once(&sem_once, once_init); + if (sem_inited!=1) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem); + errno = ENOMEM; return -1; - } else { - return 0; } + kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value); + if (ret != KERN_SUCCESS) { + fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem); + // we fail-fast here, because we have less-doc about semaphore_create for the moment + abort(); + } +#else // SEM_USE_PTHREAD + p->sem = dispatch_semaphore_create(value); + if (p->sem == NULL) { + fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#endif // SEM_USE_PTHREAD + + p->valid = 1; + + *sem = p; + + return 0; } -int tsem_wait(dispatch_semaphore_t *sem) { - dispatch_semaphore_wait(*sem, DISPATCH_TIME_FOREVER); +int tsem_wait(tsem_t *sem) { + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->val -= 1; + if (p->val < 0) { + if (pthread_cond_wait(&p->cond, &p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } return 0; +#elif defined(SEM_USE_POSIX) + return sem_wait(p->sem); +#elif defined(SEM_USE_SEM) + return semaphore_wait(p->sem); +#else // SEM_USE_PTHREAD + return dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER); +#endif // SEM_USE_PTHREAD } -int tsem_post(dispatch_semaphore_t *sem) { - dispatch_semaphore_signal(*sem); +int tsem_post(tsem_t *sem) { + if (!*sem) { + fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + struct tsem_s *p = *sem; + if (!p->valid) { + fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->val += 1; + if (p->val <= 0) { + if (pthread_cond_signal(&p->cond)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } return 0; +#elif defined(SEM_USE_POSIX) + return sem_post(p->sem); +#elif defined(SEM_USE_SEM) + return semaphore_signal(p->sem); +#else // SEM_USE_PTHREAD + return dispatch_semaphore_signal(p->sem); +#endif // SEM_USE_PTHREAD } -int tsem_destroy(dispatch_semaphore_t *sem) { +int tsem_destroy(tsem_t *sem) { + // fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem); + if (!*sem) { + // fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem); + // abort(); + return 0; + } + struct tsem_s *p = *sem; + if (!p->valid) { + // fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem); + // abort(); + return 0; + } +#ifdef SEM_USE_PTHREAD + if (pthread_mutex_lock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + p->valid = 0; + if (pthread_cond_destroy(&p->cond)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + if (pthread_mutex_unlock(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } + if (pthread_mutex_destroy(&p->lock)) { + fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem); + abort(); + } +#elif defined(SEM_USE_POSIX) + char name[NAME_MAX-4]; + snprintf(name, sizeof(name), "/t%ld", p->id); + int r = sem_unlink(name); + if (r) { + int e = errno; + fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e, strerror(e)); + abort(); + } +#elif defined(SEM_USE_SEM) + semaphore_destroy(sem_port, p->sem); +#else // SEM_USE_PTHREAD +#endif // SEM_USE_PTHREAD + + p->valid = 0; + free(p); + + *sem = NULL; return 0; } + +bool taosCheckPthreadValid(pthread_t thread) { + uint64_t id = 0; + int r = pthread_threadid_np(thread, &id); + return r ? false : true; +} + +int64_t taosGetSelfPthreadId() { + uint64_t id; + pthread_threadid_np(0, &id); + return (int64_t) id; +} + +int64_t taosGetPthreadId(pthread_t thread) { + return (int64_t)thread; +} + +void taosResetPthread(pthread_t* thread) { + *thread = NULL; +} + +bool taosComparePthread(pthread_t first, pthread_t second) { + return pthread_equal(first, second) ? true : false; +} + +int32_t taosGetPId() { + return (int32_t)getpid(); +} + +int32_t taosGetCurrentAPPName(char* name, int32_t* len) { + char buf[PATH_MAX+1]; + buf[0] = '\0'; + proc_name(getpid(), buf, sizeof(buf)-1); + buf[PATH_MAX] = '\0'; + size_t n = strlen(buf); + if (len) *len = n; + if (name) strcpy(name, buf); + return 0; +} + + diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c index f8aa02dcffd14e2def6339ccb507b9f9ab2489d1..55c07766b307f57fbc8fd76798ac880efef8a4de 100644 --- a/src/os/src/darwin/darwinSysInfo.c +++ b/src/os/src/darwin/darwinSysInfo.c @@ -18,44 +18,142 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" +#include "taoserror.h" +#include +#include + static void taosGetSystemTimezone() { - // get and set default timezone SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); + if (cfg_timezone == NULL) return; + if (cfg_timezone->cfgStatus >= TAOS_CFG_CSTATUS_DEFAULT) { + return; + } + + /* load time zone string from /etc/localtime */ + char buf[4096]; + char *tz = NULL; { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n<0) { + uError("read /etc/localtime error, reason:%s", strerror(errno)); + return; } - else { - strcpy(tsTimezone, tz); + buf[n] = '\0'; + for(int i=n-1; i>=0; --i) { + if (buf[i]=='/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } } - cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("timezone not configured, use default"); + if (!tz || 0==strchr(tz, '/')) { + uError("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", + tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], -timezone/3600); + + // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uWarn("timezone not configured, set to system default:%s", tsTimezone); } -static void taosGetSystemLocale() { - // get and set default locale +/* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ +static void taosGetSystemLocale() { // get and set default locale + char sep = '.'; + char *locale = NULL; + SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(tsLocale, "en_US.UTF-8"); + } else { tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("locale not configured, set to default:%s", tsLocale); + uWarn("locale not configured, set to system default:%s", tsLocale); } } + /* if user does not specify the charset, extract it from locale */ SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - strcpy(tsCharset, "cp936"); - cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("charset not configured, set to default:%s", tsCharset); + char *str = strrchr(tsLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); + + free(revisedCharset); + uWarn("charset not configured, set to system default:%s", tsCharset); + } else { + strcpy(tsCharset, "UTF-8"); + uWarn("can't get locale and charset from system, set it to UTF-8"); + } } } -void taosPrintOsInfo() {} +void taosPrintOsInfo() { + uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize / 1024); + // uInfo(" os openMax: %" PRId64, tsOpenMax); + // uInfo(" os streamMax: %" PRId64, tsStreamMax); + uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); + uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + + struct utsname buf; + if (uname(&buf)) { + uInfo(" can't fetch os info"); + return; + } + uInfo(" os sysname: %s", buf.sysname); + uInfo(" os nodename: %s", buf.nodename); + uInfo(" os release: %s", buf.release); + uInfo(" os version: %s", buf.version); + uInfo(" os machine: %s", buf.machine); + uInfo("=================================="); +} void taosKillSystem() { uError("function taosKillSystem, exit!"); @@ -63,12 +161,26 @@ void taosKillSystem() { } void taosGetSystemInfo() { + // taosGetProcInfos(); + + tsNumOfCores = sysconf(_SC_NPROCESSORS_ONLN); + long physical_pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGESIZE); + tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); + tsPageSize = page_size; + + // float tmp1, tmp2; + // taosGetSysMemory(&tmp1); + // taosGetProcMemory(&tmp2); + // taosGetDisk(); + // taosGetBandSpeed(&tmp1); + // taosGetCpuUsage(&tmp1, &tmp2); + // taosGetProcIO(&tmp1, &tmp2); + taosGetSystemTimezone(); taosGetSystemLocale(); } -bool taosGetDisk() { return true; } - bool taosGetProcIO(float *readKB, float *writeKB) { *readKB = 0; *writeKB = 0; @@ -101,4 +213,40 @@ int taosSystem(const char *cmd) { return -1; } -void taosSetCoreDump() {} \ No newline at end of file +void taosSetCoreDump() {} + +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { + struct statvfs info; + if (statvfs(tsDataDir, &info)) { + uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } else { + diskSize->tsize = info.f_blocks * info.f_frsize; + diskSize->avail = info.f_bavail * info.f_frsize; + diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; + return 0; + } +} + +char cmdline[1024]; + +char *taosGetCmdlineByPID(int pid) { + errno = 0; + + if (proc_pidpath(pid, cmdline, sizeof(cmdline)) <= 0) { + fprintf(stderr, "PID is %d, %s", pid, strerror(errno)); + return strerror(errno); + } + + return cmdline; +} + +bool taosGetSystemUid(char *uid) { + uuid_t uuid = {0}; + uuid_generate(uuid); + // it's caller's responsibility to make enough space for `uid`, that's 36-char + 1-null + uuid_unparse_lower(uuid, uid); + return true; +} + diff --git a/src/os/src/darwin/darwinTimer.c b/src/os/src/darwin/darwinTimer.c index 5fe65fb99ed72357d88ecbf5d4699bcbb42360c6..ee1becc91af5237296bb2a562982b33cc5c78f10 100644 --- a/src/os/src/darwin/darwinTimer.c +++ b/src/os/src/darwin/darwinTimer.c @@ -13,9 +13,82 @@ * along with this program. If not, see . */ +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + #define _DEFAULT_SOURCE #include "os.h" +#if 1 +#include + +static void (*timer_callback)(int); +static int timer_ms = 0; +static pthread_t timer_thread; +static int timer_kq = -1; +static volatile int timer_stop = 0; + +static void* timer_routine(void *arg) { + (void)arg; + + int r = 0; + struct timespec to = {0}; + to.tv_sec = timer_ms / 1000; + to.tv_nsec = (timer_ms % 1000) * 1000000; + while (!timer_stop) { + struct kevent64_s kev[10] = {0}; + r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev)/sizeof(kev[0]), 0, &to); + if (r!=0) { + fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__); + abort(); + } + timer_callback(SIGALRM); // just mock + } + + return NULL; +} + +int taosInitTimer(void (*callback)(int), int ms) { + int r = 0; + timer_ms = ms; + timer_callback = callback; + + timer_kq = kqueue(); + if (timer_kq==-1) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + + r = pthread_create(&timer_thread, NULL, timer_routine, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + return 0; +} + +void taosUninitTimer() { + int r = 0; + timer_stop = 1; + r = pthread_join(timer_thread, NULL); + if (r) { + fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__); + // since no caller of this func checks the return value for the moment + abort(); + } + close(timer_kq); + timer_kq = -1; +} + +void taos_block_sigalrm(void) { + // we don't know if there's any specific API for SIGALRM to deliver to specific thread + // this implementation relies on kqueue rather than SIGALRM +} +#else int taosInitTimer(void (*callback)(int), int ms) { signal(SIGALRM, callback); @@ -34,3 +107,17 @@ void taosUninitTimer() { setitimer(ITIMER_REAL, &tv, NULL); } +void taos_block_sigalrm(void) { + // since SIGALRM has been used + // consideration: any better solution? + static __thread int already_set = 0; + if (!already_set) { + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGALRM); + pthread_sigmask(SIG_BLOCK, &set, NULL); + already_set = 1; + } +} +#endif + diff --git a/src/os/src/darwin/eok.c b/src/os/src/darwin/eok.c new file mode 100644 index 0000000000000000000000000000000000000000..f731aaf330cb8618aa8a2dc51401ac45ecdc5440 --- /dev/null +++ b/src/os/src/darwin/eok.c @@ -0,0 +1,893 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// fail-fast or let-it-crash philosophy +// https://en.wikipedia.org/wiki/Fail-fast +// https://stackoverflow.com/questions/4393197/erlangs-let-it-crash-philosophy-applicable-elsewhere +// experimentally, we follow log-and-crash here + +#include "eok.h" + +#include "os.h" + +#include + +// #define BALANCE_CHECK_WHEN_CLOSE + +#ifdef ENABLE_LOG +#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) +#else // !ENABLE_LOG +#define D(fmt, ...) (void)fmt +#define E(fmt, ...) (void)fmt +#endif // ENABLE_LOG + +#define A(statement, fmt, ...) do { \ + if (statement) break; \ + fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + #statement, errno, strerror(errno), \ + ##__VA_ARGS__); \ + abort(); \ +} while (0) + +static int eok_dummy = 0; + +typedef struct ep_over_kq_s ep_over_kq_t; +typedef struct eok_event_s eok_event_t; + +struct ep_over_kq_s { + int kq; + + // !!! + // idx in the eoks list + // used as pseudo-file-desciptor + // must be 'closed' with epoll_close + int idx; + + ep_over_kq_t *next; + + int sv[2]; // 0 for read, 1 for write + + // all registered 'epoll events, key by fd' + int evs_count; + eok_event_t *evs_head; + eok_event_t *evs_tail; + eok_event_t *evs_free; + + // all kev changes list pending to be processed by kevent64 + // key by tuple (ident,filter), ident === fd in this case + struct kevent64_s *kchanges; + int nchanges; + int ichanges; + + // kev eventslist for kevent64 to store active events + // they remain alive among kevent64 calls + struct kevent64_s *kevslist; + int nevslist; + + pthread_mutex_t lock; + + volatile unsigned int lock_valid:1; + volatile unsigned int waiting:1; + volatile unsigned int changed:1; + volatile unsigned int wakenup:1; + volatile unsigned int stopping:1; +}; + +struct eok_event_s { + int fd; + struct epoll_event epev; + volatile unsigned int changed; // 0:registered;1:add;2:mod;3:del + + eok_event_t *next; + eok_event_t *prev; +}; + +typedef struct eoks_s eoks_t; +struct eoks_s { + pthread_mutex_t lock; + ep_over_kq_t **eoks; // note: this memory leaks when process terminates + int neoks; // we can add an extra api to let user clean + ep_over_kq_t *eoks_free_list; // currently, we just keep it simple stupid +}; + +static eoks_t eoks = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .eoks = NULL, + .neoks = 0, + .eoks_free_list = NULL, +}; + +#ifdef ENABLE_LOG +static const char* op_str(int op) { + switch (op) { + case EPOLL_CTL_ADD: return "EPOLL_CTL_ADD"; + case EPOLL_CTL_MOD: return "EPOLL_CTL_MOD"; + case EPOLL_CTL_DEL: return "EPOLL_CTL_DEL"; + default: return "UNKNOWN"; + } +} + +static __thread char buf_slots[10][1024] = {0}; +static __thread int buf_slots_linelen = sizeof(buf_slots[0])/sizeof(buf_slots[0][0]); +static __thread int buf_slots_count = sizeof(buf_slots)/(sizeof(buf_slots[0])/sizeof(buf_slots[0][0])); + +static const char* events_str(uint32_t events, int slots) { + A(slots>=0 && slots on linux + // EPOLLIN = 0x001, + // #define EPOLLIN EPOLLIN + // EPOLLPRI = 0x002, + // #define EPOLLPRI EPOLLPRI + // EPOLLOUT = 0x004, + // #define EPOLLOUT EPOLLOUT + // EPOLLRDNORM = 0x040, + // #define EPOLLRDNORM EPOLLRDNORM + // EPOLLRDBAND = 0x080, + // #define EPOLLRDBAND EPOLLRDBAND + // EPOLLWRNORM = 0x100, + // #define EPOLLWRNORM EPOLLWRNORM + // EPOLLWRBAND = 0x200, + // #define EPOLLWRBAND EPOLLWRBAND + // EPOLLMSG = 0x400, + // #define EPOLLMSG EPOLLMSG + // EPOLLERR = 0x008, + // #define EPOLLERR EPOLLERR + // EPOLLHUP = 0x010, + // #define EPOLLHUP EPOLLHUP + // EPOLLRDHUP = 0x2000, + // #define EPOLLRDHUP EPOLLRDHUP + // EPOLLEXCLUSIVE = 1u << 28, + // #define EPOLLEXCLUSIVE EPOLLEXCLUSIVE + // EPOLLWAKEUP = 1u << 29, + // #define EPOLLWAKEUP EPOLLWAKEUP + // EPOLLONESHOT = 1u << 30, + // #define EPOLLONESHOT EPOLLONESHOT + // EPOLLET = 1u << 31 + // #define EPOLLET EPOLLET +#define CHK_EV(ev) \ + if (len>0 && (events & (ev))==(ev)) { \ + n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ + p += n; \ + len -= n; \ + } + CHK_EV(EPOLLIN); + CHK_EV(EPOLLPRI); + CHK_EV(EPOLLOUT); + CHK_EV(EPOLLRDNORM); + CHK_EV(EPOLLRDBAND); + CHK_EV(EPOLLWRNORM); + CHK_EV(EPOLLWRBAND); + CHK_EV(EPOLLMSG); + CHK_EV(EPOLLERR); + CHK_EV(EPOLLHUP); + CHK_EV(EPOLLRDHUP); + CHK_EV(EPOLLEXCLUSIVE); + CHK_EV(EPOLLWAKEUP); + CHK_EV(EPOLLONESHOT); + CHK_EV(EPOLLET); +#undef CHK_EV + return buf; +} + +static const char* kev_flags_str(uint16_t flags, int slots) { + A(slots>=0 && slots + // #define EV_ADD 0x0001 /* add event to kq (implies enable) */ + // #define EV_DELETE 0x0002 /* delete event from kq */ + // #define EV_ENABLE 0x0004 /* enable event */ + // #define EV_DISABLE 0x0008 /* disable event (not reported) */ + // /* flags */ + // #define EV_ONESHOT 0x0010 /* only report one occurrence */ + // #define EV_CLEAR 0x0020 /* clear event state after reporting */ + // #define EV_RECEIPT 0x0040 /* force immediate event output */ + // /* ... with or without EV_ERROR */ + // /* ... use KEVENT_FLAG_ERROR_EVENTS */ + // /* on syscalls supporting flags */ + // #define EV_DISPATCH 0x0080 /* disable event after reporting */ + // #define EV_UDATA_SPECIFIC 0x0100 /* unique kevent per udata value */ + // #define EV_DISPATCH2 (EV_DISPATCH | EV_UDATA_SPECIFIC) + // /* ... in combination with EV_DELETE */ + // /* will defer delete until udata-specific */ + // /* event enabled. EINPROGRESS will be */ + // /* returned to indicate the deferral */ + // #define EV_VANISHED 0x0200 /* report that source has vanished */ + // /* ... only valid with EV_DISPATCH2 */ + // #define EV_SYSFLAGS 0xF000 /* reserved by system */ + // #define EV_FLAG0 0x1000 /* filter-specific flag */ + // #define EV_FLAG1 0x2000 /* filter-specific flag */ + // /* returned values */ + // #define EV_EOF 0x8000 /* EOF detected */ + // #define EV_ERROR 0x4000 /* error, data contains errno */ +#define CHK_EV(ev) \ + if (len>0 && (flags & (ev))==(ev)) { \ + n = snprintf(p, len, "%s%s", p!=buf ? "|" : "", #ev); \ + p += n; \ + len -= n; \ + } + CHK_EV(EV_ADD); + CHK_EV(EV_DELETE); + CHK_EV(EV_ENABLE); + CHK_EV(EV_DISABLE); + CHK_EV(EV_ONESHOT); + CHK_EV(EV_CLEAR); + CHK_EV(EV_RECEIPT); + CHK_EV(EV_DISPATCH); + CHK_EV(EV_UDATA_SPECIFIC); + CHK_EV(EV_DISPATCH2); + CHK_EV(EV_VANISHED); + CHK_EV(EV_SYSFLAGS); + CHK_EV(EV_FLAG0); + CHK_EV(EV_FLAG1); + CHK_EV(EV_EOF); + CHK_EV(EV_ERROR); +#undef CHK_EV + return buf; +} +#endif // ENABLE_LOG + +static ep_over_kq_t* eoks_alloc(void); +static void eoks_free(ep_over_kq_t *eok); +static ep_over_kq_t* eoks_find(int epfd); + +static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd); +static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok); +static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev); +static void eok_wakeup(ep_over_kq_t *eok); + +static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev); + +static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents); + +int epoll_create(int size) { + (void)size; + int e = 0; + ep_over_kq_t *eok = eoks_alloc(); + if (!eok) return -1; + + A(eok->kq==-1, "internal logic error"); + A(eok->lock_valid, "internal logic error"); + A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); + A(eok->sv[0]==-1, "internal logic error"); + A(eok->sv[1]==-1, "internal logic error"); + + eok->kq = kqueue(); + if (eok->kq==-1) { + e = errno; + eoks_free(eok); + errno = e; + return -1; + } + + if (socketpair(AF_LOCAL, SOCK_STREAM, 0, eok->sv)) { + e = errno; + eoks_free(eok); + errno = e; + return -1; + } + + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &eok_dummy; + D("epoll_create epfd:[%d] and sv0[%d]", eok->idx, eok->sv[0]); + if (epoll_ctl(eok->idx, EPOLL_CTL_ADD, eok->sv[0], &ev)) { + e = errno; + epoll_close(eok->idx); + errno = e; + return -1; + } + + return eok->idx; +} + +int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) { + D("epoll_ctling epfd:[%d], op:[%s], fd:[%d], events:[%04x:%s]", + epfd, op_str(op), fd, + event ? event->events : 0, + event ? events_str(event->events, 0) : "NULL"); + int e = 0; + if (epfd<0 || epfd>=eoks.neoks) { + errno = EBADF; + return -1; + } + if (fd==-1) { + errno = EBADF; + return -1; + } + if (event && !(event->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP | EPOLLOUT))) { + e = ENOTSUP; + return -1; + } + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + eok_event_t* oev = eok_find_ev(eok, fd); + if (op==EPOLL_CTL_ADD && oev) { + e = EEXIST; + break; + } + if (op!=EPOLL_CTL_ADD && !oev) { + e = ENOENT; + break; + } + if (op!=EPOLL_CTL_DEL && !event) { + e = EINVAL; + break; + } + + // prepare krev/kwev + struct kevent64_s krev = {0}; + struct kevent64_s kwev = {0}; + krev.ident = -1; + kwev.ident = -1; + uint16_t flags = 0; + // prepare internal eok event + eok_event_t ev = {0}; + ev.fd = fd; + if (event) ev.epev = *event; + struct epoll_event *pev = event; + switch (op) { + case EPOLL_CTL_ADD: { + flags = EV_ADD; + ev.changed = 1; + } break; + case EPOLL_CTL_MOD: { + flags = EV_ADD; + ev.changed = 2; + } break; + case EPOLL_CTL_DEL: { + // event is ignored + // pev points to registered epoll_event + pev = &oev->epev; + flags = EV_DELETE; + ev.changed = 3; + } break; + default: { + e = ENOTSUP; + } break; + } + + if (e) break; + + // udata will be delayed to be set + if (pev->events & (EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { + flags |= EV_EOF; + EV_SET64(&krev, ev.fd, EVFILT_READ, flags, 0, 0, -1, 0, 0); + } + if (pev->events & EPOLLOUT) { + EV_SET64(&kwev, ev.fd, EVFILT_WRITE, flags, 0, 0, -1, 0, 0); + } + + // refresh registered evlist and changelist in a transaction way + if (eok_chgs_refresh(eok, oev, &ev, &krev, &kwev)) { + e = errno; + A(e, "internal logic error"); + break; + } + eok->changed = 1; + eok_wakeup(eok); + } while (0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + + if (e) { + errno = e; + return -1; + } + + return 0; +} + +static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to); + +int epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) { + taos_block_sigalrm(); + + int e = 0; + if (!events) { + errno = EINVAL; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + return -1; + } + if (maxevents<=0) { + errno = EINVAL; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + return -1; + } + + struct timespec abstime = {0}; + A(TIME_UTC==timespec_get(&abstime, TIME_UTC), "internal logic error"); + + if (timeout!=-1) { + if (timeout<0) timeout = 0; + int64_t t = abstime.tv_nsec + timeout * 1000000; + abstime.tv_sec += t / 1000000000; + abstime.tv_nsec = t % 1000000000; + } + + int r = 0; + + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + errno = EBADF; + return -1; + } + + int cnts = 0; + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + cnts = 0; + A(eok->waiting==0, "internal logic error"); + struct kevent64_s *eventslist = eok_alloc_eventslist(eok, maxevents); + if (!eventslist) { + e = ENOMEM; + E("epoll_waiting epfd:[%d], maxevents:[%d], timeout:[%d] failed", epfd, maxevents, timeout); + break; + } + memset(eventslist, 0, maxevents * sizeof(*eventslist)); + + struct timespec now = {0}; + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + struct timespec to = do_timespec_diff(&now, &abstime); + struct timespec *pto = &to; + if (timeout==-1) { + pto = NULL; + } + + // taking the changelist + struct kevent64_s *kchanges = eok->kchanges; + int nchanges = eok->nchanges; + int ichanges = eok->ichanges; + // let outside world to add changes + eok->kchanges = NULL; + eok->nchanges = 0; + eok->ichanges = 0; + + eok->changed = 0; + eok->wakenup = 0; + eok->waiting = 1; + + A(0==pthread_mutex_unlock(&eok->lock), ""); + if (ichanges>0) { + D("kevent64 epfd[%d] changing [%d] changes and waiting...", eok->idx, ichanges); + } + errno = 0; + r = kevent64(eok->kq, kchanges, ichanges, eventslist, maxevents, 0, pto); + e = errno; + if (e) { + E("kevent64 epfd[%d] waiting done, with r[%d]", eok->idx, r); + } + A(0==pthread_mutex_lock(&eok->lock), ""); + + eok->waiting = 0; + + if (kchanges) { + if (eok->kchanges==NULL) { + // reuse + A(eok->nchanges==0 && eok->ichanges==0, "internal logic error"); + eok->kchanges = kchanges; + eok->nchanges = nchanges; + } else { + free(kchanges); + kchanges = NULL; + } + nchanges = 0; + ichanges = 0; + } + + if (r==0) { + A(timeout!=-1, "internal logic error"); + } + for (int i=0; iudata && eok->evs_head && eok->evs_tail, "internal logic error"); + eok_event_t *ev = (eok_event_t*)kev->udata; + A(kev->ident == ev->fd, "internal logic error"); + if (kev->flags & EV_ERROR) { + D("epfd[%d] error when processing change list for fd[%d], error[%s], kev_flags:[%04x:%s]", + epfd, ev->fd, strerror(kev->data), kev->flags, kev_flags_str(kev->flags, 0)); + } + switch (kev->filter) { + case EVFILT_READ: { + A((ev->epev.events & EPOLLIN), "internal logic errro"); + if (ev->epev.data.ptr==&eok_dummy) { + // it's coming from wakeup socket pair + char c = '\0'; + A(1==recv(kev->ident, &c, 1, 0), "internal logic error"); + A(0==memcmp(&c, "1", 1), "internal logic error"); + D("epfd[%d] wokenup", epfd); + continue; + } else { + if (ev->changed==3) { + D("epfd[%d] already requested to delete for fd[%d]", epfd, ev->fd); + // TODO: write a unit test for this case + // EV_DELETE? + continue; + } + // converting to epoll_event + // we shall collect all kevents for the uniq fd into one epoll_evnt + // but currently, taos never use EPOLLOUT + // just let it this way for the moment + struct epoll_event pev = {0}; + pev.data.ptr = ev->epev.data.ptr; + pev.events = EPOLLIN; + if (kev->flags & EV_EOF) { + // take all these as EOF for the moment + pev.events |= (EPOLLHUP | EPOLLERR | EPOLLRDHUP); + } + // rounded to what user care + pev.events = pev.events & ev->epev.events; + D("epfd[%d] events found for fd[%d]: [%04x:%s], which was registered: [%04x:%s], kev_flags: [%04x:%s]", + epfd, + ev->fd, pev.events, events_str(pev.events, 0), + ev->epev.events, events_str(ev->epev.events, 1), + kev->flags, kev_flags_str(kev->flags, 2)); + // now we get ev and store it + events[cnts++] = pev; + } + } break; + case EVFILT_WRITE: { + A(0, "not implemented yet"); + } break; + default: { + A(0, "internal logic error"); + } break; + } + } + if (r>=0) { + // we can safely rule out delete-requested-events from the regitered evlists + // if only changelist are correctly registered + eok_event_t *p = eok->evs_head; + while (p) { + eok_event_t *next = p->next; + if (p->changed==3) { + D("epfd[%d] removing registered event for fd[%d]: [%04x:%s]", epfd, p->fd, p->epev.events, events_str(p->epev.events, 0)); + eok_free_ev(eok, p); + } + p = next; + } + } + if (cnts==0) { + // if no user-cared-events is up + // we check to see if time is up + if (timeout!=-1) { + A(TIME_UTC==timespec_get(&now, TIME_UTC), "internal logic error"); + to = do_timespec_diff(&now, &abstime); + if (to.tv_sec==0 && to.tv_nsec==0) break; + } + // time is not up yet, continue loop + } + } while (cnts==0); + if (cnts>0) { + D("kevent64 epfd[%d] waiting done with [%d] events", epfd, cnts); + } + A(0==pthread_mutex_unlock(&eok->lock), ""); + + if (e) { + errno = e; + E("epfd[%d] epoll_wait failed", epfd); + return -1; + } + + // tell user how many events are valid + return cnts; +} + +static struct timespec do_timespec_diff(struct timespec *from, struct timespec *to) { + struct timespec delta; + delta.tv_sec = to->tv_sec - from->tv_sec; + delta.tv_nsec = to->tv_nsec - from->tv_nsec; + // norm and round up + while (delta.tv_nsec<0) { + delta.tv_sec -= 1; + delta.tv_nsec += 1000000000; + } + if (delta.tv_sec < 0) { + delta.tv_sec = 0; + delta.tv_nsec = 0; + } + return delta; +} + +int epoll_close(int epfd) { + D("epoll_closing epfd: [%d]", epfd); + ep_over_kq_t *eok = eoks_find(epfd); + if (!eok) { + errno = EBADF; + return -1; + } + + A(0==pthread_mutex_lock(&eok->lock), ""); + do { + // panic if it would be double-closed + A(eok->stopping==0, "internal logic error"); + eok->stopping = 1; + // panic if epoll_wait is pending + A(eok->waiting==0, "internal logic error"); + + if (eok->kq!=-1) { + close(eok->kq); + eok->kq = -1; + } + } while (0); + A(0==pthread_mutex_unlock(&eok->lock), ""); + eoks_free(eok); + + return 0; +} + +static struct kevent64_s* eok_alloc_eventslist(ep_over_kq_t *eok, int maxevents) { + if (maxevents<=eok->nevslist) return eok->kevslist; + struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kevslist, sizeof(*p)*maxevents); + if (!p) return NULL; + eok->kevslist = p; + eok->nevslist = maxevents; + return p; +} + +static eok_event_t* eok_find_ev(ep_over_kq_t *eok, int fd) { + eok_event_t *p = eok->evs_head; + while (p) { + if (p->fd == fd) return p; + p = p->next; + } + errno = ENOENT; + return NULL; +} + +static eok_event_t* eok_calloc_ev(ep_over_kq_t *eok) { + eok_event_t *p = NULL; + if (eok->evs_free) { + p = eok->evs_free; + eok->evs_free = p->next; + p->next = NULL; + A(p->prev==NULL, "internal logic error"); + } else { + p = (eok_event_t*)calloc(1, sizeof(*p)); + if (!p) return NULL; + A(p->next==NULL, "internal logic error"); + A(p->prev==NULL, "internal logic error"); + } + // dirty link + p->prev = eok->evs_tail; + if (eok->evs_tail) eok->evs_tail->next = p; + else eok->evs_head = p; + eok->evs_tail = p; + + eok->evs_count += 1; + + return p; +} + +static void eok_free_ev(ep_over_kq_t *eok, eok_event_t *ev) { + if (ev->prev) ev->prev->next = ev->next; + else eok->evs_head = ev->next; + if (ev->next) ev->next->prev = ev->prev; + else eok->evs_tail = ev->prev; + ev->prev = NULL; + ev->next = eok->evs_free; + eok->evs_free = ev; + + eok->evs_count -= 1; +} + +static void eok_wakeup(ep_over_kq_t *eok) { + if (!eok->waiting) return; + if (eok->wakenup) return; + eok->wakenup = 1; + A(1==send(eok->sv[1], "1", 1, 0), ""); +} + +static int eok_chgs_refresh(ep_over_kq_t *eok, eok_event_t *oev, eok_event_t *ev, struct kevent64_s *krev, struct kevent64_s *kwev) { + if (!oev) oev = eok_calloc_ev(eok); + if (!oev) return -1; + int n = 0; + if (krev->ident==ev->fd) ++n; + if (kwev->ident==ev->fd) ++n; + A(n, "internal logic error"); + if (eok->ichanges+n>eok->nchanges) { + struct kevent64_s *p = (struct kevent64_s*)realloc(eok->kchanges, sizeof(*p) * (eok->nchanges + 10)); + if (!p) { + if (ev->changed==1) { + // roll back + A(oev, "internal logic error"); + eok_free_ev(eok, oev); + } + errno = ENOMEM; + return -1; + } + eok->kchanges = p; + eok->nchanges += 10; + } + + // copy to registered event slot + oev->fd = ev->fd; + if (ev->changed!=3) { + // if add/mod, copy epoll_event + oev->epev = ev->epev; + } + oev->changed = ev->changed; + + // copy to changes list + n = 0; + if (krev->ident==ev->fd) { + krev->udata = (uint64_t)oev; + eok->kchanges[eok->ichanges++] = *krev; + ++n; + } + if (kwev->ident==ev->fd) { + kwev->udata = (uint64_t)oev; + eok->kchanges[eok->ichanges++] = *kwev; + ++n; + } + D("epfd[%d]: add #changes[%d] for fd[%d], and now #changes/registers [%d/%d]", eok->idx, n, ev->fd, eok->ichanges, eok->evs_count); + return 0; +} + +static ep_over_kq_t* eoks_alloc(void) { + ep_over_kq_t *eok = NULL; + + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + if (eoks.eoks_free_list) { + eok = eoks.eoks_free_list; + eoks.eoks_free_list = eok->next; + A(eoks.eoks, "internal logic error"); + A(eok->idx>=0 && eok->idxidx)==NULL, "internal logic error"); + *(eoks.eoks + eok->idx) = eok; + eok->next = NULL; + eok->stopping = 0; + break; + } + eok = (ep_over_kq_t*)calloc(1, sizeof(*eok)); + if (!eok) break; + eok->idx = -1; + ep_over_kq_t **ar = (ep_over_kq_t**)realloc(eoks.eoks, sizeof(**ar) * (eoks.neoks+1)); + if (!ar) break; + eoks.eoks = ar; + *(eoks.eoks + eoks.neoks) = eok; + eok->idx = eoks.neoks; + eok->kq = -1; + eok->sv[0] = -1; + eok->sv[1] = -1; + eoks.neoks += 1; + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); + + if (!eok) { + errno = ENOMEM; + return NULL; + } + if (eok->idx==-1) { + free(eok); + errno = ENOMEM; + return NULL; + } + if (eok->lock_valid) { + return eok; + } + + if (0==pthread_mutex_init(&eok->lock, NULL)) { + eok->lock_valid = 1; + return eok; + } + + eoks_free(eok); + errno = ENOMEM; + return NULL; +} + +static void eoks_free(ep_over_kq_t *eok) { + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + A(eok->idx>=0 && eok->idxnext==NULL, "internal logic error"); + + // leave eok->kchanges as is + A(eok->ichanges==0, "internal logic error"); + + A(eok->waiting==0, "internal logic error"); + eok_event_t *ev = eok->evs_head; + int sv_closed = 0; + while (ev) { + eok_event_t *next = ev->next; + if (ev->fd==eok->sv[0]) { + // fd is critical system resource + A(sv_closed==0, "internal logic error"); + close(eok->sv[0]); + eok->sv[0] = -1; + close(eok->sv[1]); + eok->sv[1] = -1; + eok_free_ev(eok, ev); + } else { + // user forget calling epoll_ctl(EPOLL_CTL_DEL) before calling epoll_close/close? + // calling close(ev->fd) here smells really bad +#ifndef BALANCE_CHECK_WHEN_CLOSE + // we just let it be and reclaim ev + eok_free_ev(eok, ev); +#else + // panic otherwise, if BALANCE_CHECK_WHEN_CLOSE is defined + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, + "epfd[%d] fd[%d]: internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?", + eok->idx, ev->fd); +#endif + } + ev = next; + } + // if (eok->evs_count==1) { + // A(eok->evs_head && eok->evs_tail && eok->evs_head==eok->evs_tail, "internal logic error"); + // A(eok->evs_head->fd==eok->sv[0] && eok->sv[0]!=-1 && eok->sv[1]!=-1, "internal logic error"); + // // fd is critical system resource + // close(eok->sv[0]); + // eok->sv[0] = -1; + // close(eok->sv[1]); + // eok->sv[1] = -1; + // eok_free_ev(eok, eok->evs_head); + // } + A(eok->evs_head==NULL && eok->evs_tail==NULL && eok->evs_count==0, + "internal logic error: have you epoll_ctl(EPOLL_CTL_DEL) everything before calling epoll_close?"); + A(eok->sv[0]==-1 && eok->sv[1]==-1, "internal logic error"); + if (eok->kq!=-1) { + close(eok->kq); + eok->kq = -1; + } + eok->next = eoks.eoks_free_list; + eoks.eoks_free_list = eok; + *(eoks.eoks + eok->idx) = NULL; + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); +} + +static ep_over_kq_t* eoks_find(int epfd) { + ep_over_kq_t *eok = NULL; + A(0==pthread_mutex_lock(&eoks.lock), ""); + do { + if (epfd<0 || epfd>=eoks.neoks) { + break; + } + A(eoks.eoks, "internal logic error"); + eok = *(eoks.eoks + epfd); + A(eok->next==NULL, "internal logic error"); + A(eok->lock_valid, "internal logic error"); + } while (0); + A(0==pthread_mutex_unlock(&eoks.lock), ""); + return eok; +} + diff --git a/src/os/src/detail/CMakeLists.txt b/src/os/src/detail/CMakeLists.txt index e4052a4fb32d4d8bda1ea05443cf95c276251e57..facfbd23af7a579ed11655ce66dddd971677fb18 100644 --- a/src/os/src/detail/CMakeLists.txt +++ b/src/os/src/detail/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(.) @@ -13,3 +13,4 @@ TARGET_LINK_LIBRARIES(osdetail os) IF (TD_ARM_32 OR TD_LINUX_32) TARGET_LINK_LIBRARIES(osdetail atomic) ENDIF () + diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index d3f0fda1a5c8535226eedbac6d5d88a1352e3c6e..144c59fa2db3ae24a81679de16116812fddbccc5 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -51,16 +51,22 @@ int taosMkDir(const char *path, mode_t mode) { return code; } -void taosRename(char* oldName, char *newName) { - // if newName in not empty, rename return fail. - // the newName must be empty or does not exist - if (rename(oldName, newName)) { + +#ifndef TAOS_OS_FUNC_DIR + +int32_t taosRename(char* oldName, char *newName) { + int32_t code = rename(oldName, newName); + if (code < 0) { uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); } else { - uInfo("successfully to rename file %s to %s", oldName, newName); + uTrace("successfully to rename file %s to %s", oldName, newName); } + + return code; } +#endif + void taosRemoveOldLogFiles(char *rootDir, int32_t keepDays) { DIR *dir = opendir(rootDir); if (dir == NULL) return; @@ -117,7 +123,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { goto cmp_end; } - int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { ret = -2; goto cmp_end; diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 2e6886aa21f8c083e1134122ac334a20edf54874..538ed378798f53c09c8cb8ada2f2a1a066d8b6bf 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -25,7 +25,8 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { char tmpPath[PATH_MAX]; int32_t len = strlen(tsTempDir); memcpy(tmpPath, tsTempDir, len); - + static uint64_t seqId = 0; + if (tmpPath[len - 1] != '/') { tmpPath[len++] = '/'; } @@ -36,8 +37,10 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, "-%d-%s"); } - char rand[8] = {0}; - taosRandStr(rand, tListLen(rand) - 1); + char rand[32] = {0}; + + sprintf(rand, "%"PRIu64, atomic_add_fetch_64(&seqId, 1)); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); } @@ -119,17 +122,49 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } +int64_t taosCopy(char *from, char *to) { + char buffer[4096]; + int fidto = -1, fidfrom = -1; + int64_t size = 0; + int64_t bytes; + + fidfrom = open(from, O_RDONLY | O_BINARY); + if (fidfrom < 0) goto _err; + + fidto = open(to, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755); + if (fidto < 0) goto _err; + + while (true) { + bytes = taosRead(fidfrom, buffer, sizeof(buffer)); + if (bytes < 0) goto _err; + if (bytes == 0) break; + + size += bytes; + + if (taosWrite(fidto, (void *)buffer, bytes) < bytes) goto _err; + if (bytes < sizeof(buffer)) break; + } + + fsync(fidto); + + close(fidfrom); + close(fidto); + return size; + +_err: + if (fidfrom >= 0) close(fidfrom); + if (fidto >= 0) close(fidto); + remove(to); + return -1; +} + #ifndef TAOS_OS_FUNC_FILE_SENDIFLE -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) { int64_t leftbytes = size; int64_t sentbytes; while (leftbytes > 0) { - /* - * TODO : Think to check if file is larger than 1GB - */ - // if (leftbytes > 1000000000) leftbytes = 1000000000; sentbytes = sendfile(dfd, sfd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c index 53310d179c0090382e009de949e5158146dc282a..291a54b6695106ba3b457d148b1439e283d6ceff 100644 --- a/src/os/src/detail/osMemory.c +++ b/src/os/src/detail/osMemory.c @@ -512,8 +512,9 @@ void * taosTRealloc(void *ptr, size_t size) { return (void *)((char *)tptr + sizeof(size_t)); } -void taosTZfree(void *ptr) { +void* taosTZfree(void* ptr) { if (ptr) { - free((void *)((char *)ptr - sizeof(size_t))); + free((void*)((char*)ptr - sizeof(size_t))); } + return NULL; } \ No newline at end of file diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index d379e56ed83bd19e4b9ff5336e95cd019f296a36..2bf2f2448752bf468497fc48a8871c5f0120b9ab 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -31,7 +31,14 @@ int tsem_wait(tsem_t* sem) { #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } -int64_t taosGetSelfPthreadId() { return (int64_t)pthread_self(); } + +int64_t taosGetSelfPthreadId() { + static __thread int id = 0; + if (id != 0) return id; + id = syscall(SYS_gettid); + return id; +} + int64_t taosGetPthreadId(pthread_t thread) { return (int64_t)thread; } void taosResetPthread(pthread_t *thread) { *thread = 0; } bool taosComparePthread(pthread_t first, pthread_t second) { return first == second; } @@ -62,4 +69,4 @@ int32_t taosGetCurrentAPPName(char *name, int32_t* len) { return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/os/src/detail/osSignal.c b/src/os/src/detail/osSignal.c new file mode 100644 index 0000000000000000000000000000000000000000..e1a0e84e7fe689bdf92a9a7bafbddb80ccd03a4b --- /dev/null +++ b/src/os/src/detail/osSignal.c @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tulog.h" + +#ifndef TAOS_OS_FUNC_SIGNAL +typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context); + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + struct sigaction act = {{0}}; +#if 1 + act.sa_flags = SA_SIGINFO; + act.sa_sigaction = (FLinuxSignalHandler)sigfp; +#else + act.sa_handler = sigfp; +#endif + sigaction(signum, &act, NULL); +} + +void taosIgnSignal(int32_t signum) { + signal(signum, SIG_IGN); +} + +void taosDflSignal(int32_t signum) { + signal(signum, SIG_DFL); +} + +#endif diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index 729471247f884977e65d86166cabf06641581e2f..db976acccd7d61223fc742b7dcdb736bea6a5f57 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -53,6 +53,16 @@ void taosBlockSIGPIPE() { } } +void taosSetMaskSIGPIPE() { + sigset_t signal_mask; + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGPIPE); + int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); + if (rc != 0) { + uError("failed to setmask SIGPIPE"); + } +} + #endif #ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT @@ -61,6 +71,9 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } +int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t* optlen) { + return getsockopt(socketfd, level, optname, optval, (socklen_t *)optlen); +} #endif #ifndef TAOS_OS_FUNC_SOCKET_INET @@ -73,4 +86,4 @@ const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } -#endif \ No newline at end of file +#endif diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index b0ca6139ed9461b12ab2845d0060b51dc388bcab..c0d46878a821db3490ccf54f8e303e37e52f84aa 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -18,6 +18,7 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" +#include "taoserror.h" #ifndef TAOS_OS_FUNC_SYSINFO @@ -45,6 +46,21 @@ static char tsProcMemFile[25] = {0}; static char tsProcIOFile[25] = {0}; static float tsPageSizeKB = 0; +static void taosGetProcInfos() { + tsPageSize = sysconf(_SC_PAGESIZE); + tsOpenMax = sysconf(_SC_OPEN_MAX); + tsStreamMax = sysconf(_SC_STREAM_MAX); + + tsProcId = (pid_t)syscall(SYS_gettid); + tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; + + snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); + snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); + snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); +} + +static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); } + bool taosGetSysMemory(float *memoryUsedMB) { float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024; *memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB; @@ -105,7 +121,8 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { } char cpu[10] = {0}; - sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); + sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, + &cpuInfo->idle); tfree(line); fclose(fp); @@ -131,7 +148,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { for (int i = 0, blank = 0; line[i] != 0; ++i) { if (line[i] == ' ') blank++; if (blank == PROCESS_ITEM) { - sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); + sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, + &cpuInfo->cutime, &cpuInfo->cstime); break; } } @@ -162,12 +180,12 @@ static void taosGetSystemTimezone() { char buf[68] = {0}; if (f != NULL) { int len = fread(buf, 64, 1, f); - if(len < 64 && ferror(f)) { + if (len < 64 && ferror(f)) { fclose(f); uError("read /etc/timezone error, reason:%s", strerror(errno)); return; } - + fclose(f); buf[sizeof(buf) - 1] = 0; @@ -231,7 +249,7 @@ static void taosGetSystemLocale() { // get and set default locale if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { locale = setlocale(LC_CTYPE, ""); if (locale == NULL) { - uError("can't get locale from system, set it to en_US.UTF-8"); + uError("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); strcpy(tsLocale, "en_US.UTF-8"); } else { tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); @@ -258,6 +276,8 @@ static void taosGetSystemLocale() { // get and set default locale } } +static int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } + bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { static uint64_t lastSysUsed = 0; static uint64_t lastSysTotal = 0; @@ -297,43 +317,18 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { return true; } -bool taosGetDisk() { +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { struct statvfs info; - const double unit = 1024 * 1024 * 1024; - - if (tscEmbedded) { - if (statvfs(tsDataDir, &info)) { - //tsTotalDataDirGB = 0; - //tsAvailDataDirGB = 0; - uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - return false; - } else { - tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); - tsAvailDataDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit); - } - } - - if (statvfs(tsLogDir, &info)) { - //tsTotalLogDirGB = 0; - //tsAvailLogDirGB = 0; - uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno)); - return false; - } else { - tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); - tsAvailLogDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit); - } - - if (statvfs("/tmp", &info)) { - //tsTotalTmpDirGB = 0; - //tsAvailTmpDirectorySpace = 0; - uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno)); - return false; + if (statvfs(tsDataDir, &info)) { + uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } else { - tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit); - tsAvailTmpDirectorySpace = (float)((double)info.f_bavail * (double)info.f_frsize / unit); + diskSize->tsize = info.f_blocks * info.f_frsize; + diskSize->avail = info.f_bavail * info.f_frsize; + diskSize->used = (info.f_blocks - info.f_bfree) * info.f_frsize; + return 0; } - - return true; } static bool taosGetCardInfo(int64_t *bytes) { @@ -344,13 +339,12 @@ static bool taosGetCardInfo(int64_t *bytes) { return false; } - size_t len = 2048; char * line = calloc(1, len); while (!feof(fp)) { memset(line, 0, len); - + int64_t rbytes = 0; int64_t rpackts = 0; int64_t tbytes = 0; @@ -465,7 +459,7 @@ bool taosGetProcIO(float *readKB, float *writeKB) { static int64_t lastReadbyte = -1; static int64_t lastWritebyte = -1; - int64_t curReadbyte = 0; + int64_t curReadbyte = 0; int64_t curWritebyte = 0; if (!taosReadProcIO(&curReadbyte, &curWritebyte)) { @@ -490,23 +484,15 @@ bool taosGetProcIO(float *readKB, float *writeKB) { } void taosGetSystemInfo() { - tsNumOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN); - tsPageSize = sysconf(_SC_PAGESIZE); - tsOpenMax = sysconf(_SC_OPEN_MAX); - tsStreamMax = sysconf(_SC_STREAM_MAX); + taosGetProcInfos(); - tsProcId = (pid_t)syscall(SYS_gettid); - tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024; - tsTotalMemoryMB = (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); - - snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId); - snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId); - snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId); + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); float tmp1, tmp2; taosGetSysMemory(&tmp1); taosGetProcMemory(&tmp2); - taosGetDisk(); + // taosGetDisk(); taosGetBandSpeed(&tmp1); taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); @@ -521,6 +507,8 @@ void taosPrintOsInfo() { uInfo(" os streamMax: %" PRId64, tsStreamMax); uInfo(" os numOfCores: %d", tsNumOfCores); uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); + uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); struct utsname buf; @@ -533,7 +521,6 @@ void taosPrintOsInfo() { uInfo(" os release: %s", buf.release); uInfo(" os version: %s", buf.version); uInfo(" os machine: %s", buf.machine); - uInfo("=================================="); } void taosKillSystem() { @@ -573,16 +560,16 @@ void taosSetCoreDump() { if (0 == tsEnableCoreFile) { return; } - + // 1. set ulimit -c unlimited struct rlimit rlim; struct rlimit rlim_new; if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - #ifndef _ALPINE +#ifndef _ALPINE uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - #else +#else uInfo("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); - #endif +#endif rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { @@ -594,57 +581,56 @@ void taosSetCoreDump() { } if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - #ifndef _ALPINE +#ifndef _ALPINE uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - #else +#else uInfo("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max); - #endif +#endif } #ifndef _TD_ARM_ // 2. set the path for saving core file struct __sysctl_args args; - int old_usespid = 0; - size_t old_len = 0; - int new_usespid = 1; - size_t new_len = sizeof(new_usespid); - + + int old_usespid = 0; + size_t old_len = 0; + int new_usespid = 1; + size_t new_len = sizeof(new_usespid); + int name[] = {CTL_KERN, KERN_CORE_USES_PID}; - + memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; args.oldlenp = &old_len; - args.newval = &new_usespid; - args.newlen = new_len; - + args.newval = &new_usespid; + args.newlen = new_len; + old_len = sizeof(old_usespid); - + if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } - - uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); + uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); old_usespid = 0; - old_len = 0; + old_len = 0; memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; + args.name = name; + args.nlen = sizeof(name) / sizeof(name[0]); + args.oldval = &old_usespid; args.oldlenp = &old_len; - + old_len = sizeof(old_usespid); - + if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } - + uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif - } bool taosGetSystemUid(char *uid) { diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index 1710c469fb48506f21d914ac586fa8356f044619..d9d070218e2a64e5f2535c073db4852b0a8960ef 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -43,7 +43,7 @@ */ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, const unsigned int day, const unsigned int hour, - const unsigned int min, const unsigned int sec) + const unsigned int min, const unsigned int sec, int64_t timezone) { unsigned int mon = mon0, year = year0; @@ -61,12 +61,6 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, res = res*24; res = ((res + hour) * 60 + min) * 60 + sec; -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - int64_t timezone = _timezone; -#endif -#endif - return (res + timezone); } @@ -219,7 +213,7 @@ int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) { /* mktime will be affected by TZ, set by using taos_options */ #ifdef WINDOWS - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); //int64_t seconds = gmtime(&tm); #else int64_t seconds = timegm(&tm); @@ -276,7 +270,13 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { return -1; } - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + int64_t timezone = _timezone; +#endif +#endif + + int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); int64_t fraction = 0; @@ -504,13 +504,13 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio int64_t end = 0; // not enough time range - if (INT64_MAX - start > pInterval->interval - 1) { + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { end = start + pInterval->interval - 1; while(end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window start += pInterval->sliding; - if (INT64_MAX - start > pInterval->interval - 1) { + if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { end = start + pInterval->interval - 1; } else { end = INT64_MAX; @@ -574,4 +574,4 @@ const char* fmtts(int64_t ts) { } return buf; -} \ No newline at end of file +} diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index 1d3ba30def9416785d72556ac218bd5a48aead77..c1135ce292c4ef0a5ae968525ca2685a97b6f62e 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -116,6 +116,9 @@ void taosUninitTimer() { pthread_sigmask(SIG_BLOCK, &set, NULL); */ void taosMsleep(int mseconds) { +#ifdef __APPLE__ + taos_block_sigalrm(); +#endif // __APPLE__ #if 1 usleep(mseconds * 1000); #else @@ -136,6 +139,7 @@ void taosMsleep(int mseconds) { /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */ #endif + } -#endif \ No newline at end of file +#endif diff --git a/src/os/src/linux/CMakeLists.txt b/src/os/src/linux/CMakeLists.txt index 8ab8f554672843eec34ed1880ba672e61e54aa7b..b1a7ebf54e58bbbdeea6d5cc219904916cc2ba03 100644 --- a/src/os/src/linux/CMakeLists.txt +++ b/src/os/src/linux/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) diff --git a/src/os/src/linux/linuxEnv.c b/src/os/src/linux/linuxEnv.c index 5772885cb43454d147dd05dc437347bbdd15f741..e3eadbc94bcf71e7cf09e70901e4de0aed427c54 100644 --- a/src/os/src/linux/linuxEnv.c +++ b/src/os/src/linux/linuxEnv.c @@ -18,7 +18,6 @@ #include "tglobal.h" void osInit() { - #ifdef _TD_POWER_ if (configDir[0] == 0) { strcpy(configDir, "/etc/power"); @@ -43,16 +42,14 @@ void osInit() { char cmdline[1024]; -char *taosGetCmdlineByPID(int pid) -{ - sprintf(cmdline, "/proc/%d/cmdline",pid); - FILE* f = fopen(cmdline,"r"); - if(f){ +char* taosGetCmdlineByPID(int pid) { + sprintf(cmdline, "/proc/%d/cmdline", pid); + FILE* f = fopen(cmdline, "r"); + if (f) { size_t size; size = fread(cmdline, sizeof(char), 1024, f); - if(size>0){ - if('\n'==cmdline[size-1]) - cmdline[size-1]='\0'; + if (size > 0) { + if ('\n' == cmdline[size - 1]) cmdline[size - 1] = '\0'; } fclose(f); } diff --git a/src/os/src/windows/CMakeLists.txt b/src/os/src/windows/CMakeLists.txt index a430dd3b3f968cd845732ead4aa1b780aea10c22..9dcc9e7e6d93ff200b7571d98724f898712658eb 100644 --- a/src/os/src/windows/CMakeLists.txt +++ b/src/os/src/windows/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) diff --git a/src/os/src/windows/wFile.c b/src/os/src/windows/wFile.c index 2204135ae657eb813144dc2bd1ae637d9510842e..3bfe1c1b5dd24185407b2e2cd1b94ff621089b0b 100644 --- a/src/os/src/windows/wFile.c +++ b/src/os/src/windows/wFile.c @@ -15,18 +15,19 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tulog.h" +#include "osSocket.h" #include "tglobal.h" +#include "tulog.h" void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { - const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; + const char *tdengineTmpFileNamePrefix = "tdengine-"; + char tmpPath[PATH_MAX]; int32_t len = (int32_t)strlen(tsTempDir); memcpy(tmpPath, tsTempDir, len); if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') { - tmpPath[len++] = '\\'; + tmpPath[len++] = '\\'; } strcpy(tmpPath + len, tdengineTmpFileNamePrefix); @@ -35,7 +36,7 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%d-%s"); } - + char rand[8] = {0}; taosRandStr(rand, tListLen(rand) - 1); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); @@ -46,18 +47,16 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { fseek(in_file, (int32_t)(*offset), 0); int64_t writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; - + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); if (rlen <= 0) { return writeLen; - } - else if (rlen < _SEND_FILE_STEP_) { + } else if (rlen < _SEND_FILE_STEP_) { fwrite(buffer, 1, rlen, out_file); return (int64_t)(writeLen + rlen); - } - else { + } else { fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); writeLen += _SEND_FILE_STEP_; } @@ -65,12 +64,44 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co int64_t remain = count - writeLen; if (remain > 0) { - size_t rlen = fread(buffer, 1, (size_t) remain, in_file); + size_t rlen = fread(buffer, 1, (size_t)remain, in_file); + if (rlen <= 0) { + return writeLen; + } else { + fwrite(buffer, 1, (size_t)remain, out_file); + writeLen += remain; + } + } + + return writeLen; +} + +int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) { + if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); + + int64_t writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = {0}; + + for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); if (rlen <= 0) { return writeLen; + } else if (rlen < _SEND_FILE_STEP_) { + taosWriteSocket(dfd, buffer, rlen); + return (int64_t)(writeLen + rlen); + } else { + taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); + writeLen += _SEND_FILE_STEP_; } - else { - fwrite(buffer, 1, (size_t) remain, out_file); + } + + int64_t remain = count - writeLen; + if (remain > 0) { + int32_t rlen = read(sfd, buffer, (int32_t)remain); + if (rlen <= 0) { + return writeLen; + } else { + taosWriteSocket(sfd, buffer, (int32_t)remain); writeLen += remain; } } @@ -78,12 +109,76 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co return writeLen; } -int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) { - uError("taosSendFile no implemented yet"); +int32_t taosFtruncate(int32_t fd, int64_t l_size) { + if (fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; + } + + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } + return 0; } -int32_t taosFtruncate(int32_t fd, int64_t length) { - uError("taosFtruncate no implemented yet"); - return 0; +int fsync(int filedes) { + if (filedes < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(filedes); + + return FlushFileBuffers(h); +} + +int32_t taosRename(char* oldName, char *newName) { + int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + if (code < 0) { + uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } else { + uTrace("successfully to rename file %s to %s", oldName, newName); + } + + return code; } \ No newline at end of file diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemphone.c index 0bc760b35e9016f63a8cd40e1035891cf36e27ae..a3f0367ee1179f74769c8883afe5ee47237ac32b 100644 --- a/src/os/src/windows/wSemphone.c +++ b/src/os/src/windows/wSemphone.c @@ -20,6 +20,7 @@ #include "ttimer.h" #include "tulog.h" #include "tutil.h" +#include bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } @@ -33,7 +34,9 @@ int64_t taosGetPthreadId(pthread_t thread) { #endif } -int64_t taosGetSelfPthreadId() { return taosGetPthreadId(pthread_self()); } +int64_t taosGetSelfPthreadId() { + return GetCurrentThreadId(); +} bool taosComparePthread(pthread_t first, pthread_t second) { return first.p == second.p; @@ -55,4 +58,4 @@ int32_t taosGetCurrentAPPName(char *name, int32_t* len) { } return 0; -} \ No newline at end of file +} diff --git a/src/os/src/windows/wSignal.c b/src/os/src/windows/wSignal.c new file mode 100644 index 0000000000000000000000000000000000000000..9de6b5f3434dc792c40b330ec37c7edd180b999e --- /dev/null +++ b/src/os/src/windows/wSignal.c @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include +#include + +typedef void (*FWinSignalHandler)(int32_t signum); + +void taosSetSignal(int32_t signum, FSignalHandler sigfp) { + if (signum == SIGUSR1) return; + + // SIGHUP doesn't exist in windows, we handle it in the way of ctrlhandler + if (signum == SIGHUP) { + SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE); + } else { + signal(signum, (FWinSignalHandler)sigfp); + } +} + +void taosIgnSignal(int32_t signum) { + if (signum == SIGUSR1 || signum == SIGHUP) return; + signal(signum, SIG_IGN); +} + +void taosDflSignal(int32_t signum) { + if (signum == SIGUSR1 || signum == SIGHUP) return; + signal(signum, SIG_DFL); +} diff --git a/src/os/src/windows/wSocket.c b/src/os/src/windows/wSocket.c index 9697c5e65fa374255c82596ec99d0c8b3e5b9aad..4e6ee15e772c5af56b4ebd11e74fc44dd57946f5 100644 --- a/src/os/src/windows/wSocket.c +++ b/src/os/src/windows/wSocket.c @@ -48,6 +48,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) { void taosIgnSIGPIPE() {} void taosBlockSIGPIPE() {} +void taosSetMaskSIGPIPE() {} int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) { if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { diff --git a/src/os/src/windows/wString.c b/src/os/src/windows/wString.c index 1fb235a00575529be660cea18c1401f1d075a49b..67237e655cd2916ec2ec80daa01e7a0f9ae0d3e3 100644 --- a/src/os/src/windows/wString.c +++ b/src/os/src/windows/wString.c @@ -75,18 +75,6 @@ char *getpass(const char *prefix) { return passwd; } -char *strndup(const char *s, size_t n) { - size_t len = strlen(s); - if (len >= n) { - len = n; - } - - char *r = calloc(len + 1, 1); - memcpy(r, s, len); - r[len] = 0; - return r; -} - int twcslen(const wchar_t *wcs) { int *wstr = (int *)wcs; if (NULL == wstr) { diff --git a/src/os/src/windows/wSysLog.c b/src/os/src/windows/wSysLog.c new file mode 100644 index 0000000000000000000000000000000000000000..866cacbabaa870a6fbe58f1a3dc363dbfdd80949 --- /dev/null +++ b/src/os/src/windows/wSysLog.c @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +void syslog(int unused, const char *format, ...) {} \ No newline at end of file diff --git a/src/os/src/windows/wSysinfo.c b/src/os/src/windows/wSysinfo.c index 1bfee25c4a6a1851bdfbc16e795185412f007df2..8a81e3079a17e013372ebd7de0facb6b49a99c7b 100644 --- a/src/os/src/windows/wSysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -21,6 +21,7 @@ #include "ttimer.h" #include "tulog.h" #include "tutil.h" +#include "taoserror.h" #if (_WIN64) #include #include @@ -31,11 +32,54 @@ #pragma comment(lib, "Mswsock.lib ") #endif +#include + #pragma warning(push) -#pragma warning(disable:4091) +#pragma warning(disable : 4091) #include #pragma warning(pop) +static int32_t taosGetTotalMemory() { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return 0; + } + + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + return (int32_t)nMemTotal; +} + +bool taosGetSysMemory(float *memoryUsedMB) { + MEMORYSTATUSEX memsStat; + memsStat.dwLength = sizeof(memsStat); + if (!GlobalMemoryStatusEx(&memsStat)) { + return false; + } + + float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); + float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); + + *memoryUsedMB = nMemTotal - nMemFree; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + unsigned bytes_used = 0; + +#if defined(_WIN64) && defined(_MSC_VER) + PROCESS_MEMORY_COUNTERS pmc; + HANDLE cur_proc = GetCurrentProcess(); + + if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { + bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); + } +#endif + + *memoryUsedMB = (float)bytes_used / 1024 / 1024; + return true; +} + static void taosGetSystemTimezone() { // get and set default timezone SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); @@ -71,38 +115,39 @@ static void taosGetSystemLocale() { } } -void taosPrintOsInfo() {} - -void taosKillSystem() { - uError("function taosKillSystem, exit!"); - exit(0); +static int32_t taosGetCpuCores() { + SYSTEM_INFO info; + GetSystemInfo(&info); + return (int32_t)info.dwNumberOfProcessors; } -void taosGetSystemInfo() { - taosGetSystemTimezone(); - taosGetSystemLocale(); +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + *sysCpuUsage = 0; + *procCpuUsage = 0; + return true; } -bool taosGetDisk() { - const double unit = 1024 * 1024 * 1024; - BOOL fResult; +int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) { unsigned _int64 i64FreeBytesToCaller; unsigned _int64 i64TotalBytes; unsigned _int64 i64FreeBytes; - char dir[4] = {'C', ':', '\\', '\0'}; - int drive_type; - - if (tscEmbedded) { - drive_type = GetDriveTypeA(dir); - if (drive_type == DRIVE_FIXED) { - fResult = GetDiskFreeSpaceExA(dir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, - (PULARGE_INTEGER)&i64FreeBytes); - if (fResult) { - tsTotalDataDirGB = tsTotalLogDirGB = tsTotalTmpDirGB = (float)(i64TotalBytes / unit); - tsAvailDataDirGB = tsAvailLogDirGB = tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit); - } - } + + BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes, + (PULARGE_INTEGER)&i64FreeBytes); + if (fResult) { + diskSize->tsize = (int64_t)(i64TotalBytes); + diskSize->avail = (int64_t)(i64FreeBytesToCaller); + diskSize->used = (int64_t)(i64TotalBytes - i64FreeBytes); + return 0; + } else { + uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + *bandSpeedKb = 0; return true; } @@ -144,48 +189,32 @@ bool taosGetProcIO(float *readKB, float *writeKB) { return true; } -bool taosGetBandSpeed(float *bandSpeedKb) { - *bandSpeedKb = 0; - return true; -} - -bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { - *sysCpuUsage = 0; - *procCpuUsage = 0; - return true; -} - -bool taosGetProcMemory(float *memoryUsedMB) { - unsigned bytes_used = 0; -#if 0 -#if defined(_WIN32) && defined(_MSC_VER) - PROCESS_MEMORY_COUNTERS pmc; - HANDLE cur_proc = GetCurrentProcess(); - - if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { - bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); - } -#endif -#endif +void taosGetSystemInfo() { + tsNumOfCores = taosGetCpuCores(); + tsTotalMemoryMB = taosGetTotalMemory(); - *memoryUsedMB = (float)bytes_used / 1024 / 1024; + float tmp1, tmp2; + // taosGetDisk(); + taosGetBandSpeed(&tmp1); + taosGetCpuUsage(&tmp1, &tmp2); + taosGetProcIO(&tmp1, &tmp2); - return true; + taosGetSystemTimezone(); + taosGetSystemLocale(); } -bool taosGetSysMemory(float *memoryUsedMB) { - MEMORYSTATUSEX memsStat; - float nMemFree; - float nMemTotal; +void taosPrintOsInfo() { + uInfo(" os numOfCores: %d", tsNumOfCores); + uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB); + uInfo(" os usedDisk: %f(GB)", tsUsedDataDirGB); + uInfo(" os availDisk: %f(GB)", tsAvailDataDirGB); + uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); + uInfo("=================================="); +} - memsStat.dwLength = sizeof(memsStat); - if (!GlobalMemoryStatusEx(&memsStat)) { - return false; - } - nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f); - nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f); - *memoryUsedMB = nMemTotal - nMemFree; - return true; +void taosKillSystem() { + uError("function taosKillSystem, exit!"); + exit(0); } int taosSystem(const char *cmd) { @@ -195,10 +224,6 @@ int taosSystem(const char *cmd) { int flock(int fd, int option) { return 0; } -int fsync(int filedes) { return 0; } - -int sigaction(int sig, struct sigaction *d, void *p) { return 0; } - LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { typedef BOOL(WINAPI * FxMiniDumpWriteDump)(IN HANDLE hProcess, IN DWORD ProcessId, IN HANDLE hFile, IN MINIDUMP_TYPE DumpType, @@ -235,4 +260,22 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { return EXCEPTION_CONTINUE_SEARCH; } -void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } \ No newline at end of file +void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); } + +bool taosGetSystemUid(char *uid) { + GUID guid; + CoCreateGuid(&guid); + + sprintf( + uid, + "%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X", + guid.Data1, guid.Data2, guid.Data3, + guid.Data4[0], guid.Data4[1], + guid.Data4[2], guid.Data4[3], + guid.Data4[4], guid.Data4[5], + guid.Data4[6], guid.Data4[7]); + + return true; +} + +char *taosGetCmdlineByPID(int pid) { return ""; } diff --git a/src/os/src/windows/wWordexp.c b/src/os/src/windows/wWordexp.c index bb9acde25a5de6c334f512fc8994edffdf5c59a3..febe22ac8fa8566607fe2bf9b0be9cc7443c8012 100644 --- a/src/os/src/windows/wWordexp.c +++ b/src/os/src/windows/wWordexp.c @@ -21,13 +21,20 @@ #include "tulog.h" #include "tutil.h" -int wordexp(const char *words, wordexp_t *pwordexp, int flags) { +int wordexp(char *words, wordexp_t *pwordexp, int flags) { pwordexp->we_offs = 0; pwordexp->we_wordc = 1; - pwordexp->we_wordv = (char **)(pwordexp->wordPos); - pwordexp->we_wordv[0] = (char *)words; + pwordexp->we_wordv[0] = pwordexp->wordPos; + + memset(pwordexp->wordPos, 0, 1025); + if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { + pwordexp->we_wordv[0] = words; + uError("failed to parse relative path:%s to abs path", words); + return -1; + } + + uTrace("parse relative path:%s to abs path:%s", words, pwordexp->wordPos); return 0; } void wordfree(wordexp_t *pwordexp) {} - diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index d03717b6e155913e4d0f9bfe442ea7dd77b47a3e..7dcaaf27e615ead75e83630788288a27e938b0a9 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -1,8 +1,8 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) ADD_SUBDIRECTORY(monitor) ADD_SUBDIRECTORY(http) -IF (TD_MQTT) +IF (TD_LINUX AND TD_MQTT) ADD_SUBDIRECTORY(mqtt) ENDIF () \ No newline at end of file diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 56c3c25d7c8522d2d31522f4d1b6dce4df1af42b..bfb47ad12e8b1ef7099109ecf5849ec3575caf5f 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) @@ -8,18 +8,16 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - -IF (TD_LINUX) - ADD_LIBRARY(http ${SRC}) - TARGET_LINK_LIBRARIES(http z) - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(http taos_static) - ELSE () - TARGET_LINK_LIBRARIES(http taos) - ENDIF () +ADD_LIBRARY(http ${SRC}) +TARGET_LINK_LIBRARIES(http z) - IF (TD_ADMIN) - TARGET_LINK_LIBRARIES(http admin) - ENDIF () +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(http taos_static) +ELSE () + TARGET_LINK_LIBRARIES(http taos) +ENDIF () + +IF (TD_ADMIN) + TARGET_LINK_LIBRARIES(http admin) ENDIF () diff --git a/src/plugins/http/inc/httpContext.h b/src/plugins/http/inc/httpContext.h index 260858c5cc22aa10dc49a61743879d768907c8fe..b016da2dd3a75bc4e51f29e3f9344458d11badea 100644 --- a/src/plugins/http/inc/httpContext.h +++ b/src/plugins/http/inc/httpContext.h @@ -22,7 +22,7 @@ bool httpInitContexts(); void httpCleanupContexts(); const char *httpContextStateStr(HttpContextState state); -HttpContext *httpCreateContext(int32_t fd); +HttpContext *httpCreateContext(SOCKET fd); bool httpInitContext(HttpContext *pContext); HttpContext *httpGetContext(void * pContext); void httpReleaseContext(HttpContext *pContext, bool clearRes); diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index ac182b707b585adf76b92cf4fbc027d22277f220..634468f3ccfd041300e0b99c728a8fb6a9b8fbae 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -16,6 +16,7 @@ #ifndef TDENGINE_HTTP_INT_H #define TDENGINE_HTTP_INT_H +#include "os.h" #include #include "pthread.h" #include "semaphore.h" @@ -39,7 +40,8 @@ #define HTTP_GC_TARGET_SIZE 512 #define HTTP_WRITE_RETRY_TIMES 500 #define HTTP_WRITE_WAIT_TIME_MS 5 -#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + TSDB_KEY_LEN) +#define HTTP_PASSWORD_LEN TSDB_UNI_LEN +#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + HTTP_PASSWORD_LEN) typedef enum HttpReqType { HTTP_REQTYPE_OTHERS = 0, @@ -139,7 +141,7 @@ typedef enum { typedef struct HttpContext { int32_t refCount; - int32_t fd; + SOCKET fd; uint32_t accessTimes; uint32_t lastAccessTime; int32_t state; @@ -147,7 +149,7 @@ typedef struct HttpContext { uint8_t parsed; char ipstr[22]; char user[TSDB_USER_LEN]; // parsed from auth token or login message - char pass[TSDB_KEY_LEN]; + char pass[HTTP_PASSWORD_LEN]; TAOS * taos; void * ppContext; HttpSession *session; @@ -166,7 +168,7 @@ typedef struct HttpThread { HttpContext * pHead; pthread_mutex_t threadMutex; bool stop; - int32_t pollFd; + EpollFd pollFd; int32_t numOfContexts; int32_t threadId; char label[HTTP_LABEL_SIZE]; @@ -177,7 +179,9 @@ typedef struct HttpServer { char label[HTTP_LABEL_SIZE]; uint32_t serverIp; uint16_t serverPort; - int32_t fd; + int8_t stop; + int8_t reserve; + SOCKET fd; int32_t numOfThreads; int32_t methodScannerLen; int32_t requestNum; diff --git a/src/plugins/http/inc/httpJson.h b/src/plugins/http/inc/httpJson.h index fcb74253b99c0cd8921d981e3ba7a4a027bf2b31..4d182d0132528367a8e25b8f72329331be2cb5c5 100644 --- a/src/plugins/http/inc/httpJson.h +++ b/src/plugins/http/inc/httpJson.h @@ -63,9 +63,11 @@ void httpJsonString(JsonBuf* buf, char* sVal, int32_t len); void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len); void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int32_t maxLen); void httpJsonInt64(JsonBuf* buf, int64_t num); +void httpJsonUInt64(JsonBuf* buf, uint64_t num); void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us); void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us); void httpJsonInt(JsonBuf* buf, int32_t num); +void httpJsonUInt(JsonBuf* buf, uint32_t num); void httpJsonFloat(JsonBuf* buf, float num); void httpJsonDouble(JsonBuf* buf, double num); void httpJsonNull(JsonBuf* buf); diff --git a/src/plugins/http/inc/httpRestJson.h b/src/plugins/http/inc/httpRestJson.h index 112e845f369c6ae3f0df76d880f84de5aeff5d8d..5f19983826a4f90bc3498ca82cc9fd495b4f300c 100644 --- a/src/plugins/http/inc/httpRestJson.h +++ b/src/plugins/http/inc/httpRestJson.h @@ -34,6 +34,8 @@ #define REST_JSON_DATA_LEN 4 #define REST_JSON_HEAD "head" #define REST_JSON_HEAD_LEN 4 +#define REST_JSON_HEAD_INFO "column_meta" +#define REST_JSON_HEAD_INFO_LEN 11 #define REST_JSON_ROWS "rows" #define REST_JSON_ROWS_LEN 4 #define REST_JSON_AFFECT_ROWS "affected_rows" @@ -51,4 +53,4 @@ bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAO bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows); void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd); -#endif \ No newline at end of file +#endif diff --git a/src/plugins/http/inc/httpUtil.h b/src/plugins/http/inc/httpUtil.h index 61cd50a77a29ad8b9a2058f4ad569493b18618ac..54c95b6980f8241c3ea6c8e563e0e42c7c737286 100644 --- a/src/plugins/http/inc/httpUtil.h +++ b/src/plugins/http/inc/httpUtil.h @@ -17,7 +17,7 @@ #define TDENGINE_HTTP_UTIL_H bool httpCheckUsedbSql(char *sql); -void httpTimeToString(time_t t, char *buf, int32_t buflen); +void httpTimeToString(int32_t t, char *buf, int32_t buflen); bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp); bool httpParseRequest(HttpContext *pContext); diff --git a/src/plugins/http/src/httpAuth.c b/src/plugins/http/src/httpAuth.c index 56da8a7089e635effa0fc54e1aeb7137ea391c23..2ef5406823b63862ba8e529ed4caea5bf7e4ec32 100644 --- a/src/plugins/http/src/httpAuth.c +++ b/src/plugins/http/src/httpAuth.c @@ -26,7 +26,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; int32_t outlen = 0; - char *base64 = (char *)base64_decode(token, len, &outlen); + char * base64 = (char *)base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token); free(base64); @@ -49,9 +49,9 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) strncpy(pContext->user, base64, (size_t)user_len); pContext->user[user_len] = 0; - char *password = user + 1; + char * password = user + 1; int32_t pass_len = (int32_t)((base64 + outlen) - password); - if (pass_len < 1 || pass_len >= TSDB_KEY_LEN) { + if (pass_len < 1 || pass_len >= HTTP_PASSWORD_LEN) { httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token); free(base64); return -1; @@ -66,14 +66,14 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) { token[len] = '\0'; - int32_t outlen = 0; + int32_t outlen = 0; unsigned char *base64 = base64_decode(token, len, &outlen); if (base64 == NULL || outlen == 0) { httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token); if (base64) free(base64); return 01; } - if (outlen != (TSDB_USER_LEN + TSDB_KEY_LEN)) { + if (outlen != (TSDB_USER_LEN + HTTP_PASSWORD_LEN)) { httpError("context:%p, fd:%d, taosd token:%s length error", pContext, pContext->fd, token); free(base64); return -1; @@ -97,14 +97,14 @@ int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) } int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) { - char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; + char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0}; size_t size = sizeof(pContext->user); tstrncpy(buffer, pContext->user, size); size = sizeof(pContext->pass); tstrncpy(buffer + sizeof(pContext->user), pContext->pass, size); - char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + TSDB_KEY_LEN); - char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + TSDB_KEY_LEN); + char *encrypt = taosDesEncode(KEY_DES_4, buffer, TSDB_USER_LEN + HTTP_PASSWORD_LEN); + char *base64 = base64_encode((const unsigned char *)encrypt, TSDB_USER_LEN + HTTP_PASSWORD_LEN); size_t len = strlen(base64); tstrncpy(token, base64, len + 1); diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index 266228e7ac7df7641b9166e07ddba9bad7968461..13f706af653e9c5c1b9fb0a4c602355b001d0cac 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -35,20 +35,24 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; if (pContext->fd >= 0) { epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); - int32_t fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); +#ifdef WINDOWS + SOCKET fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); +#else + SOCKET fd = atomic_val_compare_exchange_64(&pContext->fd, pContext->fd, -1); +#endif taosCloseSocket(fd); } } static void httpDestroyContext(void *data) { HttpContext *pContext = *(HttpContext **)data; - if (pContext->fd > 0) taosClose(pContext->fd); + if (pContext->fd > 0) taosCloseSocket(pContext->fd); HttpThread *pThread = pContext->pThread; httpRemoveContextFromEpoll(pContext); httpReleaseSession(pContext); atomic_sub_fetch_32(&pThread->numOfContexts, 1); - + httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount, data, pContext->pThread->label, pContext->pThread->numOfContexts); pContext->pThread = 0; @@ -100,15 +104,13 @@ const char *httpContextStateStr(HttpContextState state) { } } -void httpNotifyContextClose(HttpContext *pContext) { - shutdown(pContext->fd, SHUT_WR); -} +void httpNotifyContextClose(HttpContext *pContext) { shutdown(pContext->fd, SHUT_WR); } bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) { return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState); } -HttpContext *httpCreateContext(int32_t fd) { +HttpContext *httpCreateContext(SOCKET fd) { HttpContext *pContext = calloc(1, sizeof(HttpContext)); if (pContext == NULL) return NULL; @@ -123,8 +125,8 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); - // set the ref to 0 - taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false); + // set the ref to 0 + taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false); return pContext; } @@ -152,7 +154,10 @@ void httpReleaseContext(HttpContext *pContext, bool clearRes) { } if (clearRes) { - httpClearParser(pContext->parser); + if (pContext->parser) { + httpClearParser(pContext->parser); + } + memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); } HttpContext **ppContext = pContext->ppContext; @@ -174,7 +179,6 @@ bool httpInitContext(HttpContext *pContext) { pContext->encodeMethod = NULL; memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd)); - httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed); return true; } @@ -184,9 +188,9 @@ void httpCloseContextByApp(HttpContext *pContext) { pContext->parsed = false; bool keepAlive = true; - if (parser->httpVersion == HTTP_VERSION_10 && parser->keepAlive != HTTP_KEEPALIVE_ENABLE) { + if (parser && parser->httpVersion == HTTP_VERSION_10 && parser->keepAlive != HTTP_KEEPALIVE_ENABLE) { keepAlive = false; - } else if (parser->httpVersion != HTTP_VERSION_10 && parser->keepAlive == HTTP_KEEPALIVE_DISABLE) { + } else if (parser && parser->httpVersion != HTTP_VERSION_10 && parser->keepAlive == HTTP_KEEPALIVE_DISABLE) { keepAlive = false; } else { } diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index c21799e736027eb4a021032d670f1b542500fe8f..925c74e7cd46751ada1e04873710a329e77bded0 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -59,11 +59,11 @@ bool gcGetUserFromUrl(HttpContext* pContext) { bool gcGetPassFromUrl(HttpContext* pContext) { HttpParser* pParser = pContext->parser; - if (pParser->path[GC_PASS_URL_POS].pos >= TSDB_KEY_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) { + if (pParser->path[GC_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, TSDB_KEY_LEN); + tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, HTTP_PASSWORD_LEN); return true; } @@ -192,7 +192,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) { break; } - cJSON* alias = cJSON_GetObjectItem(query, "alias"); + cJSON* alias = cJSON_GetObjectItem(query, "alias"); int32_t aliasBuffer = -1; if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) { aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring); diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 2c9eca11deb51acc8cc25965b83cc686ac04c6e5..8c223a15000d47becaf92fde34c92edb3f7ce707 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -86,7 +86,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; - int32_t num_fields = taos_num_fields(result); + int32_t num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); if (num_fields == 0) { return false; @@ -101,7 +101,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, // such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d) int32_t dataFields = -1; int32_t groupFields = -1; - bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; + bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP; if (hasTimestamp) { dataFields = 1; if (num_fields > 2) groupFields = num_fields - 1; @@ -125,15 +125,15 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, cmd->numOfRows--; continue; } - int32_t* length = taos_fetch_lengths(result); + int32_t *length = taos_fetch_lengths(result); // for group by if (groupFields != -1) { - char target[HTTP_GC_TARGET_SIZE] = {0}; + char target[HTTP_GC_TARGET_SIZE] = {0}; int32_t len; - len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer); - for (int32_t i = dataFields + 1; istate) { + switch (gzip->state) { case EHTTP_GZIP_READY: { inflateEnd(gzip->gzip); } break; - default: break; + default: + break; } if (gzip->gzip) { free(gzip->gzip); @@ -60,43 +59,43 @@ static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) { gzip->state = EHTTP_GZIP_CLOSED; } -ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { - ehttp_gzip_t *gzip = (ehttp_gzip_t*)calloc(1, sizeof(*gzip)); +ehttp_gzip_t *ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) { + ehttp_gzip_t *gzip = (ehttp_gzip_t *)calloc(1, sizeof(*gzip)); if (!gzip) return NULL; do { - gzip->conf = conf; - gzip->callbacks = callbacks; - gzip->arg = arg; + gzip->conf = conf; + gzip->callbacks = callbacks; + gzip->arg = arg; if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data; - gzip->gzip = (z_stream*)calloc(1, sizeof(*gzip->gzip)); + gzip->gzip = (z_stream *)calloc(1, sizeof(*gzip->gzip)); if (gzip->conf.get_header) { - gzip->header = (gz_header*)calloc(1, sizeof(*gzip->header)); + gzip->header = (gz_header *)calloc(1, sizeof(*gzip->header)); } - if (gzip->conf.chunk_size<=0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; - gzip->chunk = (char*)malloc(gzip->conf.chunk_size); + if (gzip->conf.chunk_size <= 0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT; + gzip->chunk = (char *)malloc(gzip->conf.chunk_size); if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break; - gzip->gzip->zalloc = Z_NULL; - gzip->gzip->zfree = Z_NULL; - gzip->gzip->opaque = Z_NULL; - - // 863 windowBits can also be greater than 15 for optional gzip decoding. Add - // 864 32 to windowBits to enable zlib and gzip decoding with automatic header - // 865 detection, or add 16 to decode only the gzip format (the zlib format will - // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a - // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see - // 868 below), inflate() will not automatically decode concatenated gzip streams. - // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state - // 870 would need to be reset to continue decoding a subsequent gzip stream. - int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS + gzip->gzip->zalloc = Z_NULL; + gzip->gzip->zfree = Z_NULL; + gzip->gzip->opaque = Z_NULL; + + // 863 windowBits can also be greater than 15 for optional gzip decoding. Add + // 864 32 to windowBits to enable zlib and gzip decoding with automatic header + // 865 detection, or add 16 to decode only the gzip format (the zlib format will + // 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + // 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + // 868 below), inflate() will not automatically decode concatenated gzip streams. + // 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state + // 870 would need to be reset to continue decoding a subsequent gzip stream. + int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS if (ret != Z_OK) break; if (gzip->header) { ret = inflateGetHeader(gzip->gzip, gzip->header); } if (ret != Z_OK) break; - gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; gzip->state = EHTTP_GZIP_READY; return gzip; } while (0); @@ -105,7 +104,7 @@ ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_ return NULL; } -ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); +ehttp_gzip_t *ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg); void ehttp_gzip_destroy(ehttp_gzip_t *gzip) { ehttp_gzip_cleanup(gzip); @@ -129,16 +128,16 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { } if (ret != Z_OK && ret != Z_STREAM_END) return -1; - if (gzip->gzip->avail_out>0) { - if (ret!=Z_STREAM_END) continue; + if (gzip->gzip->avail_out > 0) { + if (ret != Z_STREAM_END) continue; } - int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = (z_const Bytef*)gzip->chunk; - gzip->gzip->avail_out = gzip->conf.chunk_size; + gzip->gzip->next_out = (z_const Bytef *)gzip->chunk; + gzip->gzip->avail_out = gzip->conf.chunk_size; } return 0; @@ -147,21 +146,20 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) { int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) { if (gzip->state != EHTTP_GZIP_READY) return -1; - gzip->gzip->next_in = NULL; - gzip->gzip->avail_in = 0; + gzip->gzip->next_in = NULL; + gzip->gzip->avail_in = 0; int32_t ret; ret = inflate(gzip->gzip, Z_FINISH); if (ret != Z_STREAM_END) return -1; - int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk; + int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk); gzip->gzip->next_out[0] = '\0'; gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len); - gzip->gzip->next_out = NULL; - gzip->gzip->avail_out = 0; + gzip->gzip->next_out = NULL; + gzip->gzip->avail_out = 0; return 0; } - diff --git a/src/plugins/http/src/httpHandle.c b/src/plugins/http/src/httpHandle.c index 7c565075147c03cc2d65055c5d4d4773424ab927..ad79e24061d251351bc73790ac341c9dc31b29b6 100644 --- a/src/plugins/http/src/httpHandle.c +++ b/src/plugins/http/src/httpHandle.c @@ -48,7 +48,7 @@ bool httpProcessData(HttpContext* pContext) { /* * httpCloseContextByApp has been called when parsing the error */ - //httpCloseContextByApp(pContext); + // httpCloseContextByApp(pContext); } else { httpProcessRequest(pContext); } diff --git a/src/plugins/http/src/httpJson.c b/src/plugins/http/src/httpJson.c index 1aa6cfac4bf9414e6aa8004ca1bb120b2d40feba..19166e720f0e517ccd1a793742836ae4bc245bd1 100644 --- a/src/plugins/http/src/httpJson.c +++ b/src/plugins/http/src/httpJson.c @@ -44,20 +44,21 @@ int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t int32_t writeLen = 0; do { - if (pContext->fd > 2){ + if (pContext->fd > 2) { len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL); - } - else { + } else { return sz; } if (len < 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), countWait); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), + countWait); if (++countWait > HTTP_WRITE_RETRY_TIMES) break; taosMsleep(HTTP_WRITE_WAIT_TIME_MS); continue; } else if (len == 0) { - httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, strerror(errno)); + httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, + strerror(errno)); break; } else { countWait = 0; @@ -80,7 +81,7 @@ int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz) return writeSz; } -int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32_t sz) { +int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz) { int32_t writeSz = httpWriteBufByFd(pContext, buf, sz); if (writeSz != sz) { httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz, @@ -92,8 +93,8 @@ int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32 int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { int32_t remain = 0; - char sLen[24]; - uint64_t srcLen = (uint64_t) (buf->lst - buf->buf); + char sLen[24]; + int32_t srcLen = (int32_t)(buf->lst - buf->buf); if (buf->pContext->fd <= 0) { httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd); @@ -113,21 +114,21 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd); return 0; // there is no data to dump. } else { - int32_t len = sprintf(sLen, "%" PRIx64 "\r\n", srcLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd, - srcLen, buf->buf); + int32_t len = sprintf(sLen, "%x\r\n", srcLen); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, + buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); - remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int32_t)srcLen); + remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen); } } else { - char compressBuf[JSON_BUFFER_SIZE] = {0}; + char compressBuf[JSON_BUFFER_SIZE] = {0}; int32_t compressBufLen = JSON_BUFFER_SIZE; int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast); if (ret == 0) { if (compressBufLen > 0) { int32_t len = sprintf(sLen, "%x\r\n", compressBufLen); - httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s", - buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); + httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext, + buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf); httpWriteBufNoTrace(buf->pContext, sLen, len); remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen); } else { @@ -136,7 +137,7 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) { remain = 0; // there is no data to dump. } } else { - httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s", + httpError("context:%p, fd:%d, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf); remain = 0; } @@ -154,8 +155,8 @@ void httpWriteJsonBufHead(JsonBuf* buf) { buf->pContext->fd = -1; } - char msg[1024] = {0}; - int32_t len = -1; + char msg[1024] = {0}; + int32_t len = -1; if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) { len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion], @@ -255,17 +256,23 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) { buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRId64, num); } +void httpJsonUInt64(JsonBuf* buf, uint64_t num) { + httpJsonItemToken(buf); + httpJsonTestBuf(buf, MAX_NUM_STR_SZ); + buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%" PRIu64, num); +} + void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { - char ts[35] = {0}; - struct tm *ptm; - int32_t precision = 1000; + char ts[35] = {0}; + struct tm* ptm; + int32_t precision = 1000; if (us) { precision = 1000000; } time_t tt = t / precision; ptm = localtime(&tt); - int32_t length = (int32_t) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); + int32_t length = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm); if (us) { length += snprintf(ts + length, 8, ".%06" PRId64, t % precision); } else { @@ -276,9 +283,9 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) { } void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us) { - char ts[40] = {0}; - struct tm *ptm; - int32_t precision = 1000; + char ts[40] = {0}; + struct tm* ptm; + int32_t precision = 1000; if (us) { precision = 1000000; } @@ -302,6 +309,12 @@ void httpJsonInt(JsonBuf* buf, int32_t num) { buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%d", num); } +void httpJsonUInt(JsonBuf* buf, uint32_t num) { + httpJsonItemToken(buf); + httpJsonTestBuf(buf, MAX_NUM_STR_SZ); + buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%u", num); +} + void httpJsonFloat(JsonBuf* buf, float num) { httpJsonItemToken(buf); httpJsonTestBuf(buf, MAX_NUM_STR_SZ); diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index b84483453736aac7693c9ec671ec5a27c4178072..18cea56cfe6e0d1e8a4f833a920e5473536e1c7e 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -110,7 +110,7 @@ static void httpCleanupString(HttpString *str) { static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { if (str->size == 0) { str->pos = 0; - str->size = 64; + str->size = len + 1; str->str = malloc(str->size); } else if (str->pos + len + 1 >= str->size) { str->size += len; @@ -130,7 +130,7 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { static void httpClearString(HttpString *str) { if (str->str) { str->str[0] = '\0'; - str->pos = 0; + str->pos = 0; } } @@ -153,7 +153,7 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target for (int32_t i = 0; i < HTTP_MAX_URL; i++) { char *pSeek = strchr(pStart, '/'); if (pSeek == NULL) { - (void)httpAppendString(pParser->path + i, pStart, strlen(pStart)); + (void)httpAppendString(pParser->path + i, pStart, (int32_t)strlen(pStart)); break; } else { (void)httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart)); @@ -229,7 +229,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const return 0; } - else if (strncasecmp(key, "Connection: ", 12) == 0) { + else if (strncasecmp(key, "Connection", 10) == 0) { if (strncasecmp(val, "Keep-Alive", 10) == 0) { parser->keepAlive = HTTP_KEEPALIVE_ENABLE; } else { @@ -237,7 +237,6 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive); } - #if 0 else if (0 == strcasecmp(key, "Content-Encoding")) { if (0 == strcmp(val, "gzip")) { @@ -246,7 +245,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } return 0; } - #endif +#endif else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) { if (strstr(val, "gzip")) { @@ -272,20 +271,17 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const } else if (0 == strcasecmp(key, "Authorization")) { - char * t = NULL; - char * s = NULL; + char t[6] = {0}; + char s[129] = {0}; int32_t bytes = 0; - int32_t n = sscanf(val, "%ms %ms%n", &t, &s, &bytes); - if (n == 2 && t && s && bytes == strlen(val)) { + int32_t n = sscanf(val, "%5s %128s%n", t, s, &bytes); + if (n == 2 && t[0] && s[0] && bytes == strlen(val)) { if (strcmp(t, "Basic") == 0) { free(parser->authContent); - parser->authContent = s; + parser->authContent = strdup(s); parser->authType = HTTP_BASIC_AUTH; - s = NULL; - free(t); - free(s); httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_BASIC_AUTH); return -1; @@ -293,13 +289,10 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const return 0; } else if (strcmp(t, "Taosd") == 0) { free(parser->authContent); - parser->authContent = s; + parser->authContent = strdup(s); parser->authType = HTTP_TAOSD_AUTH; - s = NULL; - free(t); - free(s); httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent); - int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, strlen(parser->authContent)); + int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent)); if (ok != 0) { httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_TAOSD_AUTH); return -1; @@ -309,16 +302,12 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_TYPE); - free(t); - free(s); return -1; } } else { parser->authType = HTTP_INVALID_AUTH; httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s); httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_FORMAT); - free(t); - free(s); return -1; } } @@ -349,7 +338,7 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) { newSize = MIN(newSize, HTTP_BUFFER_SIZE); buf->str = realloc(buf->str, newSize); buf->size = newSize; - + if (buf->str == NULL) { httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, buf->size); httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); @@ -410,9 +399,7 @@ static int32_t httpPopStack(HttpParser *parser) { return 0; } -static void httpClearStack(HttpStack *stack) { - stack->pos = 0; -} +static void httpClearStack(HttpStack *stack) { stack->pos = 0; } static int32_t httpCleanupStack(HttpStack *stack) { free(stack->stacks); @@ -451,7 +438,7 @@ void httpInitParser(HttpParser *parser) { free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; free(parser->authContent); parser->authContent = NULL; - + httpClearStack(&parser->stacks); httpClearString(&parser->str); httpClearString(&parser->body); @@ -497,7 +484,7 @@ void httpDestroyParser(HttpParser *parser) { free(parser->key); parser->key = NULL; free(parser->val); parser->val = NULL; free(parser->authContent); parser->authContent = NULL; - + httpCleanupStack(&parser->stacks); httpCleanupString(&parser->str); httpCleanupString(&parser->body); @@ -513,42 +500,53 @@ void httpDestroyParser(HttpParser *parser) { free(parser); } -#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) +#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c)) char *httpDecodeUrl(const char *enc) { - int32_t ok = 1; + int32_t ok = 1; HttpString str = {0}; while (*enc) { char *p = strchr(enc, '%'); if (!p) break; int32_t hex, cnt; - int32_t n = sscanf(p+1, "%2x%n", &hex, &cnt); - if (n!=1 && cnt !=2) { ok = 0; break; } - if (httpAppendString(&str, enc, p-enc)) { ok = 0; break; } + int32_t n = sscanf(p + 1, "%2x%n", &hex, &cnt); + if (n != 1 && cnt != 2) { + ok = 0; + break; + } + if (httpAppendString(&str, enc, (int32_t)(p - enc))) { + ok = 0; + break; + } char c = (char)hex; - if (httpAppendString(&str, &c, 1)) { ok = 0; break; } - enc = p+3; + if (httpAppendString(&str, &c, 1)) { + ok = 0; + break; + } + enc = p + 3; } char *dec = NULL; if (ok && *enc) { - if (httpAppendString(&str, enc, strlen(enc))) { ok = 0; } + if (httpAppendString(&str, enc, (int32_t)strlen(enc))) { + ok = 0; + } } if (ok) { dec = str.str; str.str = NULL; } - httpCleanupString(&str); + //httpCleanupString(&str); return dec; } static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) { - HttpParser *parser = (HttpParser*)arg; + HttpParser *parser = (HttpParser *)arg; httpOnBody(parser, buf, len); } static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == 'G' || c == 'P' || c == 'H' || c == 'D' || c == 'C' || c == 'O' || c == 'T') { if (httpAppendString(&parser->str, &c, 1)) { @@ -570,7 +568,7 @@ static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, co static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (parser->str.pos == 1) { if (c == 'T' && parser->str.str[0] == 'H') { @@ -608,7 +606,7 @@ static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STAT static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -637,7 +635,7 @@ static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (!isspace(c) && c != '\r' && c != '\n') { if (httpAppendString(&parser->str, &c, 1)) { @@ -648,7 +646,7 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c } break; } - parser->target = strdup(parser->str.str); + parser->target = httpDecodeUrl(parser->str.str); if (!parser->target) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; @@ -664,10 +662,10 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { const char *prefix = "HTTP/1."; - int32_t len = strlen(prefix); + int32_t len = (int32_t)strlen(prefix); if (parser->str.pos < len) { if (prefix[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); @@ -717,6 +715,12 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, if (parser->method) { ok = httpOnRequestLine(parser, parser->method, parser->target, parser->version); + /* + if (parser->target) { + free(parser->target); + parser->target = NULL; + } + */ } httpClearString(&parser->str); @@ -727,7 +731,7 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == ' ') { httpPopStack(parser); @@ -742,7 +746,7 @@ static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isdigit(c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -767,7 +771,7 @@ static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE stat static int32_t httpParserOnReasonPhrase(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == '\r') { parser->reasonPhrase = strdup(parser->str.str); @@ -808,10 +812,10 @@ static int32_t httpParserPostProcess(HttpParser *parser) { static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { - const char *s = "\r\n"; - int32_t len = strlen(s); + const char *s = "\r\n"; + int32_t len = (int32_t)strlen(s); if (s[parser->str.pos] != c) { httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); ok = -1; @@ -838,7 +842,7 @@ static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, con static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c == '\r') { httpPopStack(parser); @@ -876,7 +880,7 @@ static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, c static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -888,7 +892,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state break; } if (c == ':') { - parser->key = strdup(parser->str.str); + parser->key = strdup(parser->str.str); if (!parser->key) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); ok = -1; @@ -908,7 +912,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.pos > 0)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -935,10 +939,10 @@ static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; - int32_t bytes; - int32_t len; - int32_t n; + int32_t ok = 0; + int32_t bytes; + int32_t len; + int32_t n; do { if (isxdigit(c)) { if (httpAppendString(&parser->str, &c, 1)) { @@ -985,7 +989,7 @@ static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { if (httpAppendString(&parser->str, &c, 1)) { httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c); @@ -1019,7 +1023,7 @@ static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, co static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) { HttpContext *pContext = parser->pContext; - int32_t ok = 0; + int32_t ok = 0; do { ok = -1; httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c); @@ -1029,8 +1033,8 @@ static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, cons } static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { - HttpContext *pContext = parser->pContext; - int32_t ok = 0; + HttpContext * pContext = parser->pContext; + int32_t ok = 0; HTTP_PARSER_STATE state = httpTopStack(parser); do { if (state == HTTP_PARSER_BEGIN) { @@ -1119,9 +1123,9 @@ static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) { int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len) { HttpContext *pContext = parser->pContext; - const char *p = buf; - int32_t ret = 0; - int32_t i = 0; + const char * p = buf; + int32_t ret = 0; + int32_t i = 0; while (i < len) { int32_t again = 0; diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 1c039abb4d4cd546f6673648217a2410f57ae8e7..7f7ce404600b96b0efe8385ea0aed7be412a6ba0 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -38,16 +38,16 @@ typedef struct { } SHttpWorkerPool; typedef struct { - void * param; - void * result; - int32_t code; - int32_t rows; + void * param; + void * result; + int32_t code; + int32_t rows; FHttpResultFp fp; } SHttpResult; static SHttpWorkerPool tsHttpPool; -static taos_qset tsHttpQset; -static taos_queue tsHttpQueue; +static taos_qset tsHttpQset; +static taos_queue tsHttpQueue; void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) { if (tsHttpQueue != NULL) { @@ -59,7 +59,9 @@ void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int3 pMsg->fp = fp; taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg); } else { - (*fp)(param, result, code, rows); + taos_stop_query(result); + taos_free_result(result); + //(*fp)(param, result, code, rows); } } @@ -105,7 +107,7 @@ static bool httpAllocateResultQueue() { httpDebug("http result worker:%d is launched, total:%d", pWorker->workerId, tsHttpPool.num); } - httpInfo("http result queue is opened"); + httpInfo("http result queue is opened"); return true; } @@ -134,14 +136,14 @@ void httpCleanupResultQueue() { for (int32_t i = 0; i < tsHttpPool.num; ++i) { SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsHttpQset); } } for (int32_t i = 0; i < tsHttpPool.num; ++i) { SHttpWorker *pWorker = tsHttpPool.httpWorker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } } diff --git a/src/plugins/http/src/httpResp.c b/src/plugins/http/src/httpResp.c index 72604e79b710f09e67fde825915db63bc62243d9..063f2bb04e4760d683434d9e6a600791c2a57888 100644 --- a/src/plugins/http/src/httpResp.c +++ b/src/plugins/http/src/httpResp.c @@ -74,7 +74,7 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) { httpCode = 404; else if (errNo == TSDB_CODE_HTTP_UNSUPPORT_URL) httpCode = 404; - else if (errNo == TSDB_CODE_HTTP_INVLALID_URL) + else if (errNo == TSDB_CODE_HTTP_INVALID_URL) httpCode = 404; else if (errNo == TSDB_CODE_HTTP_NO_ENOUGH_MEMORY) httpCode = 507; @@ -160,8 +160,9 @@ void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) { if (temp[i] == '\"') { temp[i] = '\''; } else if (temp[i] == '\n') { - temp[i] = ' '; - } else {} + temp[i] = ' '; + } else { + } } httpSendErrorRespImp(pContext, httpCode, "Bad Request", TSDB_CODE_TSC_INVALID_SQL & 0XFFFF, temp); diff --git a/src/plugins/http/src/httpRestHandle.c b/src/plugins/http/src/httpRestHandle.c index 66f1db1a541322a0018e741dd537cd7b518f4bb5..8728b9e37a1561dd545c6878756dcfbc909a27fd 100644 --- a/src/plugins/http/src/httpRestHandle.c +++ b/src/plugins/http/src/httpRestHandle.c @@ -72,11 +72,11 @@ bool restGetUserFromUrl(HttpContext* pContext) { bool restGetPassFromUrl(HttpContext* pContext) { HttpParser* pParser = pContext->parser; - if (pParser->path[REST_PASS_URL_POS].pos >= TSDB_KEY_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) { + if (pParser->path[REST_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) { return false; } - tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, TSDB_KEY_LEN); + tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, HTTP_PASSWORD_LEN); return true; } @@ -95,7 +95,6 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) { return false; } - /* * for async test * @@ -142,6 +141,6 @@ bool restProcessRequest(struct HttpContext* pContext) { } else { } - httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVLALID_URL); + httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_URL); return false; } diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index a5b156bffc23ea3cdb042f456a327895fc0bdaa6..60c23e603e5107e858db2138d1aed87d5f8a5e1b 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -35,7 +35,7 @@ void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t // data row array end httpJsonToken(jsonBuf, JsonArrEnd); - cmd->numOfRows = affect_rows; + cmd->numOfRows = 1; } void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) { @@ -75,6 +75,44 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) // head array end httpJsonToken(jsonBuf, JsonArrEnd); + // column_meta begin + httpJsonItemToken(jsonBuf); + httpJsonPairHead(jsonBuf, REST_JSON_HEAD_INFO, REST_JSON_HEAD_INFO_LEN); + // column_meta array begin + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + if (num_fields == 0) { + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + httpJsonItemToken(jsonBuf); + httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, TSDB_DATA_TYPE_INT); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, 4); + + httpJsonToken(jsonBuf, JsonArrEnd); + } else { + for (int32_t i = 0; i < num_fields; ++i) { + httpJsonItemToken(jsonBuf); + httpJsonToken(jsonBuf, JsonArrStt); + + httpJsonItemToken(jsonBuf); + httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name)); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, fields[i].type); + httpJsonItemToken(jsonBuf); + httpJsonInt(jsonBuf, fields[i].bytes); + + httpJsonToken(jsonBuf, JsonArrEnd); + } + } + + // column_meta array end + httpJsonToken(jsonBuf, JsonArrEnd); + // data begin httpJsonItemToken(jsonBuf); httpJsonPairHead(jsonBuf, REST_JSON_DATA, REST_JSON_DATA_LEN); @@ -83,7 +121,8 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) httpJsonToken(jsonBuf, JsonArrStt); } -bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, int32_t timestampFormat) { +bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, + int32_t timestampFormat) { JsonBuf *jsonBuf = httpMallocJsonBuf(pContext); if (jsonBuf == NULL) return false; @@ -95,7 +134,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, if (row == NULL) { continue; } - int32_t* length = taos_fetch_lengths(result); + int32_t *length = taos_fetch_lengths(result); // data row array begin httpJsonItemToken(jsonBuf); @@ -123,6 +162,18 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, case TSDB_DATA_TYPE_BIGINT: httpJsonInt64(jsonBuf, *((int64_t *)row[i])); break; + case TSDB_DATA_TYPE_UTINYINT: + httpJsonUInt(jsonBuf, *((uint8_t *)row[i])); + break; + case TSDB_DATA_TYPE_USMALLINT: + httpJsonUInt(jsonBuf, *((uint16_t *)row[i])); + break; + case TSDB_DATA_TYPE_UINT: + httpJsonUInt(jsonBuf, *((uint32_t *)row[i])); + break; + case TSDB_DATA_TYPE_UBIGINT: + httpJsonUInt64(jsonBuf, *((uint64_t *)row[i])); + break; case TSDB_DATA_TYPE_FLOAT: httpJsonFloat(jsonBuf, GET_FLOAT_VAL(row[i])); break; @@ -131,15 +182,17 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - httpJsonStringForTransMean(jsonBuf, (char*)row[i], length[i]); + httpJsonStringForTransMean(jsonBuf, (char *)row[i], length[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: if (timestampFormat == REST_TIMESTAMP_FMT_LOCAL_STRING) { - httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); + httpJsonTimestamp(jsonBuf, *((int64_t *)row[i]), + taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); } else if (timestampFormat == REST_TIMESTAMP_FMT_TIMESTAMP) { httpJsonInt64(jsonBuf, *((int64_t *)row[i])); } else { - httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); + httpJsonUtcTimestamp(jsonBuf, *((int64_t *)row[i]), + taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO); } break; default: @@ -148,8 +201,8 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } // data row array end - httpJsonToken(jsonBuf, JsonArrEnd); - cmd->numOfRows ++; + httpJsonToken(jsonBuf, JsonArrEnd); + cmd->numOfRows++; if (pContext->fd <= 0) { httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user); @@ -168,15 +221,15 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, } bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP); } bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING); } bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) { - return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); + return restBuildSqlJson(pContext, cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING); } void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd) { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 4896d50c6cabf6d924161c97f7fe020fe81e2ebf..a5f40fdc4cf89f6322d310ca2093336862a9045d 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -25,34 +25,70 @@ #include "httpResp.h" #include "httpUtil.h" -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif - static bool httpReadData(HttpContext *pContext); -static void httpStopThread(HttpThread* pThread) { +#ifdef __APPLE__ +static int sv_dummy = 0; +#endif + +static void httpStopThread(HttpThread *pThread) { pThread->stop = true; // signal the thread to stop, try graceful method first, // and use pthread_cancel when failed - struct epoll_event event = { .events = EPOLLIN }; - eventfd_t fd = eventfd(1, 0); +#ifdef __APPLE__ + int sv[2]; + sv[0] = sv[1] = -1; + int r = socketpair(PF_LOCAL, SOCK_STREAM, 0, sv); + do { + if (r) break; + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &sv_dummy; + pThread->stop = true; + r = epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, sv[0], &ev); + if (r) break; + if (1 != send(sv[1], "1", 1, 0)) { + r = -1; + break; + } + } while (0); + if (r) { + pthread_cancel(pThread->thread); + } +#else + struct epoll_event event = {.events = EPOLLIN}; + eventfd_t fd = eventfd(1, 0); if (fd == -1) { - httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", + pThread->label, strerror(errno)); pThread->stop = true; pthread_cancel(pThread->thread); } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", + pThread->label, strerror(errno)); pthread_cancel(pThread->thread); } +#endif // __APPLE__ pthread_join(pThread->thread, NULL); + +#ifdef __APPLE__ + if (sv[0] != -1) { + close(sv[0]); + sv[0] = -1; + } + if (sv[1] != -1) { + close(sv[1]); + sv[1] = -1; + } +#else // __APPLE__ if (fd != -1) { - close(fd); + taosCloseSocket(fd); } +#endif // __APPLE__ - close(pThread->pollFd); + EpollClose(pThread->pollFd); pthread_mutex_destroy(&(pThread->threadMutex)); } @@ -60,9 +96,12 @@ void httpCleanUpConnect() { HttpServer *pServer = &tsHttpServer; if (pServer->pThreads == NULL) return; - pthread_join(pServer->thread, NULL); + if (taosCheckPthreadValid(pServer->thread)) { + pthread_join(pServer->thread, NULL); + } + for (int32_t i = 0; i < pServer->numOfThreads; ++i) { - HttpThread* pThread = pServer->pThreads + i; + HttpThread *pThread = pServer->pThreads + i; if (pThread != NULL) { httpStopThread(pThread); } @@ -72,15 +111,12 @@ void httpCleanUpConnect() { } static void httpProcessHttpData(void *param) { - HttpServer *pServer = &tsHttpServer; - HttpThread *pThread = (HttpThread *)param; + HttpServer * pServer = &tsHttpServer; + HttpThread * pThread = (HttpThread *)param; HttpContext *pContext; int32_t fdNum; - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - pthread_sigmask(SIG_SETMASK, &set, NULL); + taosSetMaskSIGPIPE(); while (1) { struct epoll_event events[HTTP_MAX_EVENTS]; @@ -93,11 +129,20 @@ static void httpProcessHttpData(void *param) { if (fdNum <= 0) continue; for (int32_t i = 0; i < fdNum; ++i) { +#ifdef __APPLE__ + if (events[i].data.ptr == &sv_dummy) { + // no need to drain the recv buffer of sv[0] + // since there's only one time to send at most 1 byte to sv[0] + // btw, pThread->stop shall be already set, thus never reached here + httpDebug("if you see this line, there's internal logic error"); + continue; + } +#endif // __APPLE__ pContext = httpGetContext(events[i].data.ptr); if (pContext == NULL) { httpError("context:%p, is already released, close connect", events[i].data.ptr); - //epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); - //taosClose(events[i].data.fd); + // epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + // taosClose(events[i].data.fd); continue; } @@ -154,7 +199,7 @@ static void httpProcessHttpData(void *param) { } static void *httpAcceptHttpConnection(void *arg) { - int32_t connFd = -1; + SOCKET connFd = -1; struct sockaddr_in clientAddr; int32_t threadId = 0; HttpServer * pServer = &tsHttpServer; @@ -162,10 +207,7 @@ static void *httpAcceptHttpConnection(void *arg) { HttpContext * pContext = NULL; int32_t totalFds = 0; - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - pthread_sigmask(SIG_SETMASK, &set, NULL); + taosSetMaskSIGPIPE(); pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort); @@ -180,7 +222,12 @@ static void *httpAcceptHttpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(clientAddr); - connFd = (int32_t)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + connFd = accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen); + if (pServer->stop) { + httpDebug("http server:%s socket stop, exiting...", pServer->label); + break; + } + if (connFd == -1) { if (errno == EINVAL) { httpDebug("http server:%s socket was shutdown, exiting...", pServer->label); @@ -202,7 +249,7 @@ static void *httpAcceptHttpConnection(void *arg) { taosCloseSocket(connFd); continue; } -#endif +#endif taosKeepTcpAlive(connFd); taosSetNonblocking(connFd, 1); @@ -212,22 +259,22 @@ static void *httpAcceptHttpConnection(void *arg) { pContext = httpCreateContext(connFd); if (pContext == NULL) { - httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, taosInetNtoa(clientAddr.sin_addr), - htons(clientAddr.sin_port)); + httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, + taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); taosCloseSocket(connFd); continue; } pContext->pThread = pThread; sprintf(pContext->ipstr, "%s:%u", taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); - + struct epoll_event event; event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP; event.data.ptr = pContext; if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, pContext->ipstr, pThread->label, strerror(errno)); - taosClose(pContext->fd); + taosCloseSocket(pContext->fd); httpReleaseContext(pContext, true); continue; } @@ -242,7 +289,7 @@ static void *httpAcceptHttpConnection(void *arg) { threadId = threadId % pServer->numOfThreads; } - close(pServer->fd); + taosCloseSocket(pServer->fd); return NULL; } @@ -265,8 +312,8 @@ bool httpInitConnect() { return false; } - pThread->pollFd = epoll_create(HTTP_MAX_EVENTS); // size does not matter - if (pThread->pollFd < 0) { + pThread->pollFd = (EpollFd)epoll_create(HTTP_MAX_EVENTS); // size does not matter + if (pThread->pollFd <= 0) { httpError("http thread:%s, failed to create HTTP epoll", pThread->label); pthread_mutex_destroy(&(pThread->threadMutex)); return false; @@ -278,7 +325,7 @@ bool httpInitConnect() { if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) { httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label, strerror(errno)); - pthread_mutex_destroy(&(pThread->threadMutex)); + pthread_mutex_destroy(&(pThread->threadMutex)); return false; } pthread_attr_destroy(&thattr); @@ -309,7 +356,7 @@ static bool httpReadData(HttpContext *pContext) { } if (pParser->parsed) { - httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); + httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed); return false; } diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 35ce0160b20b8cec0126f5f2415a878ccb072d0e..2e1ee7df2fe53d835ee5f3b61bd7619a9121edce 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -35,7 +35,8 @@ void httpCreateSession(HttpContext *pContext, void *taos) { session.refCount = 1; int32_t len = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); - pContext->session = taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); + pContext->session = + taosCachePut(server->sessionCache, session.id, len, &session, sizeof(HttpSession), tsHttpSessionExpire * 1000); // void *temp = pContext->session; // taosCacheRelease(server->sessionCache, (void **)&temp, false); @@ -56,7 +57,7 @@ static void httpFetchSessionImp(HttpContext *pContext) { HttpServer *server = &tsHttpServer; pthread_mutex_lock(&server->serverMutex); - char sessionId[HTTP_SESSION_ID_LEN]; + char sessionId[HTTP_SESSION_ID_LEN]; int32_t len = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass); pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len); diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index cc8e9e86e37dea8de50d624435662c1e21ff4a94..4e9b54b7bd2b25b2a8bbdd29e6d54ae314b7519d 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -64,7 +64,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32 } taos_free_result(result); - + if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) { (encode->stopJsonFp)(pContext, singleCmd); } @@ -82,7 +82,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; - HttpSqlCmds *multiCmds = pContext->multiCmds; + HttpSqlCmds * multiCmds = pContext->multiCmds; HttpEncodeMethod *encode = pContext->encodeMethod; HttpSqlCmd *singleCmd = multiCmds->cmds + multiCmds->pos; @@ -269,8 +269,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code pContext->user, tstrerror(code), pObj, taos_errstr(pObj)); httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj)); } else { - httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, - pContext->user, tstrerror(code), pObj); + httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user, + tstrerror(code), pObj); httpSendErrorResp(pContext, code); } taos_free_result(result); @@ -381,7 +381,7 @@ void httpExecCmd(HttpContext *pContext) { void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) { HttpContext *pContext = param; taos_free_result(result); - + if (pContext == NULL) return; if (code < 0) { diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 34a70a658b49085312d046fe8799ac75b733c5d8..203db218953b6d9859a4e60b589ce01e730e2773 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -89,7 +89,17 @@ int32_t httpStartSystem() { void httpStopSystem() { tsHttpServer.status = HTTP_SERVER_CLOSING; + tsHttpServer.stop = 1; +#ifdef WINDOWS + closesocket(tsHttpServer.fd); +#elif __APPLE__ + if (tsHttpServer.fd!=-1) { + close(tsHttpServer.fd); + tsHttpServer.fd = -1; + } +#else shutdown(tsHttpServer.fd, SHUT_RD); +#endif tgCleanupHandle(); } @@ -105,10 +115,8 @@ void httpCleanUpSystem() { pthread_mutex_destroy(&tsHttpServer.serverMutex); tfree(tsHttpServer.pThreads); tsHttpServer.pThreads = NULL; - + tsHttpServer.status = HTTP_SERVER_CLOSED; } -int32_t httpGetReqCount() { - return atomic_exchange_32(&tsHttpServer.requestNum, 0); -} +int32_t httpGetReqCount() { return atomic_exchange_32(&tsHttpServer.requestNum, 0); } diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index 0a3ef539e37acd576596e5444ea3989602c00e90..c1d006ff5a0c2cb5037ee6833b58d2ba749196e5 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -137,8 +137,8 @@ void tgInitSchemas(int32_t size) { } void tgParseSchemaMetric(cJSON *metric) { - STgSchema schema = {0}; - bool parsedOk = true; + STgSchema schema = {0}; + bool parsedOk = true; // name cJSON *name = cJSON_GetObjectItem(metric, "name"); @@ -186,7 +186,7 @@ void tgParseSchemaMetric(cJSON *metric) { schema.tbName = calloc(tbnameLen + 1, 1); strcpy(schema.tbName, tbname->valuestring); - // fields + // fields cJSON *fields = cJSON_GetObjectItem(metric, "fields"); if (fields == NULL) { goto ParseEnd; @@ -227,14 +227,14 @@ ParseEnd: } } -int32_t tgParseSchema(const char *content, char*fileName) { +int32_t tgParseSchema(const char *content, char *fileName) { cJSON *root = cJSON_Parse(content); if (root == NULL) { httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content); return -1; } int32_t size = 0; - cJSON *metrics = cJSON_GetObjectItem(root, "metrics"); + cJSON * metrics = cJSON_GetObjectItem(root, "metrics"); if (metrics != NULL) { size = cJSON_GetArraySize(metrics); if (size <= 0) { @@ -276,8 +276,8 @@ int32_t tgReadSchema(char *fileName) { rewind(fp); char * content = (char *)calloc(contentSize + 1, 1); - int32_t result = fread(content, 1, contentSize, fp); - + int32_t result = (int32_t)fread(content, 1, contentSize, fp); + if (result != contentSize) { httpError("failed to read telegraf schema file:%s", fileName); fclose(fp); @@ -296,7 +296,7 @@ int32_t tgReadSchema(char *fileName) { } void tgInitHandle(HttpServer *pServer) { - char fileName[TSDB_FILENAME_LEN*2] = {0}; + char fileName[TSDB_FILENAME_LEN * 2] = {0}; sprintf(fileName, "%s/taos.telegraf.cfg", configDir); if (tgReadSchema(fileName) <= 0) { tgFreeSchemas(); @@ -308,9 +308,7 @@ void tgInitHandle(HttpServer *pServer) { httpAddMethod(pServer, &tgDecodeMethod); } -void tgCleanupHandle() { - tgFreeSchemas(); -} +void tgCleanupHandle() { tgFreeSchemas(); } bool tgGetUserFromUrl(HttpContext *pContext) { HttpParser *pParser = pContext->parser; @@ -324,7 +322,7 @@ bool tgGetUserFromUrl(HttpContext *pContext) { bool tgGetPassFromUrl(HttpContext *pContext) { HttpParser *pParser = pContext->parser; - if (pParser->path[TG_PASS_URL_POS].pos >= TSDB_KEY_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) { + if (pParser->path[TG_PASS_URL_POS].pos >= HTTP_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) { return false; } @@ -357,7 +355,7 @@ char *tgGetStableName(char *stname, cJSON *fields, int32_t fieldsSize) { bool schemaMatched = true; for (int32_t f = 0; f < schema->fieldNum; ++f) { char *fieldName = schema->fields[f]; - bool fieldMatched = false; + bool fieldMatched = false; for (int32_t i = 0; i < fieldsSize; i++) { cJSON *field = cJSON_GetArrayItem(fields, i); @@ -469,9 +467,9 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { } /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ + * tag size may be larget than TSDB_COL_NAME_LEN + * we keep the first TSDB_COL_NAME_LEN bytes + */ if (0) { if (strlen(tag->string) >= TSDB_COL_NAME_LEN) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_SIZE); @@ -540,9 +538,9 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { return false; } /* - * tag size may be larget than TSDB_COL_NAME_LEN - * we keep the first TSDB_COL_NAME_LEN bytes - */ + * tag size may be larget than TSDB_COL_NAME_LEN + * we keep the first TSDB_COL_NAME_LEN bytes + */ if (0) { if (strlen(field->string) >= TSDB_COL_NAME_LEN) { httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE); @@ -578,8 +576,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { table_cmd->cmdType = HTTP_CMD_TYPE_INSERT; // order by tag name - cJSON *orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; - int32_t orderTagsLen = 0; + cJSON * orderedTags[TG_MAX_SORT_TAG_SIZE] = {0}; + int32_t orderTagsLen = 0; for (int32_t i = 0; i < tagsSize; ++i) { cJSON *tag = cJSON_GetArrayItem(tags, i); orderedTags[orderTagsLen++] = tag; @@ -603,7 +601,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { if (tsTelegrafUseFieldNum == 0) { table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s", stname); } else { - table_cmd->stable = stable_cmd->stable = httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); + table_cmd->stable = stable_cmd->stable = + httpAddToSqlCmdBuffer(pContext, "%s_%d_%d", stname, fieldsSize, orderTagsLen); } table_cmd->stable = stable_cmd->stable = httpShrinkTableName(pContext, table_cmd->stable, httpGetCmdsString(pContext, table_cmd->stable)); @@ -627,9 +626,11 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // table name if (tsTelegrafUseFieldNum == 0) { - table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); + table_cmd->table = stable_cmd->table = + httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%s", stname, host->valuestring); } else { - table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); + table_cmd->table = stable_cmd->table = + httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring); } for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; diff --git a/src/plugins/http/src/httpUtil.c b/src/plugins/http/src/httpUtil.c index 39168ee96dc0ef83dc329614f63543d2c6c7fdfa..399a33954d5670a7e928f856a898dcde1e4ac4eb 100644 --- a/src/plugins/http/src/httpUtil.c +++ b/src/plugins/http/src/httpUtil.c @@ -29,7 +29,7 @@ bool httpCheckUsedbSql(char *sql) { return false; } -void httpTimeToString(time_t t, char *buf, int32_t buflen) { +void httpTimeToString(int32_t t, char *buf, int32_t buflen) { memset(buf, 0, (size_t)buflen); char ts[32] = {0}; @@ -37,7 +37,7 @@ void httpTimeToString(time_t t, char *buf, int32_t buflen) { time_t tt = t / 1000; ptm = localtime(&tt); strftime(ts, 31, "%Y-%m-%d %H:%M:%S", ptm); - sprintf(buf, "%s.%03ld", ts, t % 1000); + sprintf(buf, "%s.%03d", ts, t % 1000); } int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...) { @@ -160,8 +160,7 @@ bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferS free(multiCmds->cmds); multiCmds->cmds = (HttpSqlCmd *)malloc((size_t)cmdSize * sizeof(HttpSqlCmd)); if (multiCmds->cmds == NULL) { - httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, - pContext->user, cmdSize); + httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize); return false; } multiCmds->maxSize = (int16_t)cmdSize; @@ -240,6 +239,10 @@ JsonBuf *httpMallocJsonBuf(HttpContext *pContext) { pContext->jsonBuf = (JsonBuf *)malloc(sizeof(JsonBuf)); } + if (!pContext->jsonBuf->pContext) { + pContext->jsonBuf->pContext = pContext; + } + return pContext->jsonBuf; } @@ -350,74 +353,78 @@ char *httpGetCmdsString(HttpContext *pContext, int32_t pos) { } int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) { - int32_t err = 0; + int32_t err = 0; z_stream gzipStream = {0}; static char dummyHead[2] = { - 0x8 + 0x7 * 0x10, - (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, + 0x8 + 0x7 * 0x10, + (((0x8 + 0x7 * 0x10) * 0x100 + 30) / 31 * 31) & 0xFF, }; - gzipStream.zalloc = (alloc_func) 0; - gzipStream.zfree = (free_func) 0; - gzipStream.opaque = (voidpf) 0; - gzipStream.next_in = (Bytef *) srcData; + gzipStream.zalloc = (alloc_func)0; + gzipStream.zfree = (free_func)0; + gzipStream.opaque = (voidpf)0; + gzipStream.next_in = (Bytef *)srcData; gzipStream.avail_in = 0; - gzipStream.next_out = (Bytef *) destData; + gzipStream.next_out = (Bytef *)destData; if (inflateInit2(&gzipStream, 47) != Z_OK) { return -1; } while (gzipStream.total_out < *nDestData && gzipStream.total_in < nSrcData) { - gzipStream.avail_in = gzipStream.avail_out = nSrcData; //1 + gzipStream.avail_in = gzipStream.avail_out = nSrcData; // 1 if ((err = inflate(&gzipStream, Z_NO_FLUSH)) == Z_STREAM_END) { break; } if (err != Z_OK) { if (err == Z_DATA_ERROR) { - gzipStream.next_in = (Bytef *) dummyHead; + gzipStream.next_in = (Bytef *)dummyHead; gzipStream.avail_in = sizeof(dummyHead); if ((err = inflate(&gzipStream, Z_NO_FLUSH)) != Z_OK) { return -2; } - } else return -3; + } else { + return -3; + } } } if (inflateEnd(&gzipStream) != Z_OK) { return -4; } - *nDestData = gzipStream.total_out; + *nDestData = (int32_t)gzipStream.total_out; return 0; } int32_t httpGzipCompressInit(HttpContext *pContext) { - pContext->gzipStream.zalloc = (alloc_func) 0; - pContext->gzipStream.zfree = (free_func) 0; - pContext->gzipStream.opaque = (voidpf) 0; - if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + pContext->gzipStream.zalloc = (alloc_func)0; + pContext->gzipStream.zfree = (free_func)0; + pContext->gzipStream.opaque = (voidpf)0; + if (deflateInit2(&pContext->gzipStream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16, 8, Z_DEFAULT_STRATEGY) != + Z_OK) { return -1; } return 0; } -int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) { +int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, + bool isTheLast) { int32_t err = 0; - int32_t lastTotalLen = (int32_t) (pContext->gzipStream.total_out); - pContext->gzipStream.next_in = (Bytef *) srcData; - pContext->gzipStream.avail_in = (uLong) nSrcData; - pContext->gzipStream.next_out = (Bytef *) destData; - pContext->gzipStream.avail_out = (uLong) (*nDestData); + int32_t lastTotalLen = (int32_t)(pContext->gzipStream.total_out); + pContext->gzipStream.next_in = (Bytef *)srcData; + pContext->gzipStream.avail_in = (uLong)nSrcData; + pContext->gzipStream.next_out = (Bytef *)destData; + pContext->gzipStream.avail_out = (uLong)(*nDestData); while (pContext->gzipStream.avail_in != 0) { if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) { return -1; } - int32_t cacheLen = pContext->gzipStream.total_out - lastTotalLen; + int32_t cacheLen = (int32_t)(pContext->gzipStream.total_out - lastTotalLen); if (cacheLen >= *nDestData) { return -2; } @@ -442,12 +449,12 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, } } - *nDestData = (int32_t) (pContext->gzipStream.total_out) - lastTotalLen; + *nDestData = (int32_t)(pContext->gzipStream.total_out) - lastTotalLen; return 0; } -bool httpUrlMatch(HttpContext* pContext, int32_t pos, char* cmp) { - HttpParser* pParser = pContext->parser; +bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp) { + HttpParser *pParser = pContext->parser; if (pos < 0 || pos >= HTTP_MAX_URL) { return false; diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index 90189c9d75b16a3689a8b6447d4ece4eb9766e1b..28c62a099c0f2bea8b33a57c577bc89c7fb15aaa 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) @@ -6,12 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(./src SRC) -IF (TD_LINUX) - ADD_LIBRARY(monitor ${SRC}) - - IF (TD_SOMODE_STATIC) - TARGET_LINK_LIBRARIES(monitor taos_static) - ELSE () - TARGET_LINK_LIBRARIES(monitor taos) - ENDIF () -ENDIF () +ADD_LIBRARY(monitor ${SRC}) + +IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(monitor taos_static) +ELSE () + TARGET_LINK_LIBRARIES(monitor taos) +ENDIF () diff --git a/src/plugins/monitor/src/monMain.c b/src/plugins/monitor/src/monMain.c index 9443b1ce12deba75a070f25ae4f67f7887870f7d..94af8e3ecd6623f791f6ad797176a59ca2605c2b 100644 --- a/src/plugins/monitor/src/monMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -20,7 +20,6 @@ #include "tlog.h" #include "ttimer.h" #include "tutil.h" -#include "tsystem.h" #include "tscUtil.h" #include "tsclient.h" #include "dnode.h" @@ -79,8 +78,8 @@ int32_t monInitSystem() { strcpy(tsMonitor.ep, tsLocalEp); } - int len = strlen(tsMonitor.ep); - for (int i = 0; i < len; ++i) { + int32_t len = (int32_t)strlen(tsMonitor.ep); + for (int32_t i = 0; i < len; ++i) { if (tsMonitor.ep[i] == ':' || tsMonitor.ep[i] == '-' || tsMonitor.ep[i] == '.') { tsMonitor.ep[i] = '_'; } @@ -104,7 +103,9 @@ int32_t monInitSystem() { } int32_t monStartSystem() { - taos_init(); + if (taos_init()) { + return -1; + } tsMonitor.start = 1; monExecuteSQLFp = monExecuteSQL; monInfo("monitor module start"); @@ -148,7 +149,7 @@ static void *monThreadFunc(void *param) { } if (tsMonitor.state == MON_STATE_NOT_INIT) { - int code = 0; + int32_t code = 0; for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) { monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex); @@ -246,7 +247,10 @@ void monStopSystem() { void monCleanupSystem() { tsMonitor.quiting = 1; monStopSystem(); - pthread_join(tsMonitor.thread, NULL); + if (taosCheckPthreadValid(tsMonitor.thread)) { + pthread_join(tsMonitor.thread, NULL); + } + if (tsMonitor.conn != NULL) { taos_close(tsMonitor.conn); tsMonitor.conn = NULL; @@ -288,7 +292,7 @@ static int32_t monBuildCpuSql(char *sql) { // unit is GB static int32_t monBuildDiskSql(char *sql) { - return sprintf(sql, ", %f, %d", (tsTotalDataDirGB - tsAvailDataDirGB), (int32_t)tsTotalDataDirGB); + return sprintf(sql, ", %f, %d", tsUsedDataDirGB, (int32_t)tsTotalDataDirGB); } // unit is Kb @@ -330,7 +334,7 @@ static void monSaveSystemInfo() { pos += monBuildReqSql(sql + pos); void *res = taos_query(tsMonitor.conn, tsMonitor.sql); - int code = taos_errno(res); + int32_t code = taos_errno(res); taos_free_result(res); if (code != 0) { diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt index 3761f70134501a5db7b7d5080e5a5bcf5f592858..50b0bbe8af4faeab41a7b041d6aa51747f0aab3e 100644 --- a/src/plugins/mqtt/CMakeLists.txt +++ b/src/plugins/mqtt/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) @@ -18,3 +18,15 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(mqtt taos) ENDIF () ENDIF () + +IF (TD_DARWIN) + ADD_LIBRARY(mqtt ${SRC}) + TARGET_LINK_LIBRARIES(mqtt cJson mqttc) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(mqtt taos_static) + ELSE () + TARGET_LINK_LIBRARIES(mqtt taos) + ENDIF () +ENDIF () + diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index e403251858bdea2c89cc050ab3dbe3a5110843e3..f23ac7dd86932ba42dde7c2891865f7dff546a00 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) @@ -14,3 +14,8 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(query m rt) ADD_SUBDIRECTORY(tests) ENDIF () + +IF (TD_DARWIN) + TARGET_LINK_LIBRARIES(query m) + ADD_SUBDIRECTORY(tests) +ENDIF () diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/qAggMain.h similarity index 78% rename from src/query/inc/tsqlfunction.h rename to src/query/inc/qAggMain.h index 51048bbe72368cb6869092b65d5f035f26f3545b..c59067c4b357aa084cd6f31c39f3038075d18a82 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/qAggMain.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSQLFUNCTION_H -#define TDENGINE_TSQLFUNCTION_H +#ifndef TDENGINE_QAGGMAIN_H +#define TDENGINE_QAGGMAIN_H #ifdef __cplusplus extern "C" { @@ -26,6 +26,7 @@ extern "C" { #include "taosdef.h" #include "trpc.h" #include "tvariant.h" +#include "tsdb.h" #define TSDB_FUNC_INVALID_ID -1 #define TSDB_FUNC_COUNT 0 @@ -59,30 +60,34 @@ extern "C" { #define TSDB_FUNC_FIRST_DST 25 #define TSDB_FUNC_LAST_DST 26 -#define TSDB_FUNC_INTERP 27 - -#define TSDB_FUNC_RATE 28 -#define TSDB_FUNC_IRATE 29 -#define TSDB_FUNC_SUM_RATE 30 -#define TSDB_FUNC_SUM_IRATE 31 -#define TSDB_FUNC_AVG_RATE 32 -#define TSDB_FUNC_AVG_IRATE 33 - -#define TSDB_FUNC_TID_TAG 34 -#define TSDB_FUNC_HISTOGRAM 35 -#define TSDB_FUNC_HLL 36 -#define TSDB_FUNC_MODE 37 -#define TSDB_FUNC_SAMPLE 38 -#define TSDB_FUNC_CEIL 39 -#define TSDB_FUNC_FLOOR 40 -#define TSDB_FUNC_ROUND 41 -#define TSDB_FUNC_MAVG 42 -#define TSDB_FUNC_CSUM 43 +#define TSDB_FUNC_STDDEV_DST 27 +#define TSDB_FUNC_INTERP 28 + +#define TSDB_FUNC_RATE 29 +#define TSDB_FUNC_IRATE 30 +#define TSDB_FUNC_SUM_RATE 31 +#define TSDB_FUNC_SUM_IRATE 32 +#define TSDB_FUNC_AVG_RATE 33 +#define TSDB_FUNC_AVG_IRATE 34 + +#define TSDB_FUNC_TID_TAG 35 +#define TSDB_FUNC_BLKINFO 36 + +#define TSDB_FUNC_HISTOGRAM 37 +#define TSDB_FUNC_HLL 38 +#define TSDB_FUNC_MODE 39 +#define TSDB_FUNC_SAMPLE 40 +#define TSDB_FUNC_CEIL 41 +#define TSDB_FUNC_FLOOR 42 +#define TSDB_FUNC_ROUND 43 +#define TSDB_FUNC_MAVG 44 +#define TSDB_FUNC_CSUM 45 + #define TSDB_FUNCSTATE_SO 0x1u // single output #define TSDB_FUNCSTATE_MO 0x2u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM #define TSDB_FUNCSTATE_STREAM 0x4u // function avail for stream -#define TSDB_FUNCSTATE_STABLE 0x8u // function avail for metric +#define TSDB_FUNCSTATE_STABLE 0x8u // function avail for super table #define TSDB_FUNCSTATE_OF 0x10u // outer forward #define TSDB_FUNCSTATE_NEED_TS 0x20u // timestamp is required during query processing #define TSDB_FUNCSTATE_SELECTIVITY 0x40u // selectivity functions, can exists along with tag columns @@ -90,19 +95,12 @@ extern "C" { #define TSDB_BASE_FUNC_SO TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF #define TSDB_BASE_FUNC_MO TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_STABLE | TSDB_FUNCSTATE_OF - #define TSDB_FUNCTIONS_NAME_MAX_LENGTH 16 #define TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE 50 #define DATA_SET_FLAG ',' // to denote the output area has data, not null value #define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG) -#define QUERY_COND_REL_PREFIX_IN "IN|" -#define QUERY_COND_REL_PREFIX_LIKE "LIKE|" - -#define QUERY_COND_REL_PREFIX_IN_LEN 3 -#define QUERY_COND_REL_PREFIX_LIKE_LEN 5 - #define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 @@ -153,7 +151,7 @@ typedef struct SResultRowCellInfo { typedef struct SPoint1 { int64_t key; - double val; + union{double val; char* ptr;}; } SPoint1; #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) @@ -171,8 +169,8 @@ typedef struct SExtTagsInfo { // sql function runtime context typedef struct SQLFunctionCtx { - int32_t startOffset; int32_t size; // number of rows + void * pInput; // input data buffer uint32_t order; // asc|desc int16_t inputType; int16_t inputBytes; @@ -181,15 +179,14 @@ typedef struct SQLFunctionCtx { int16_t outputBytes; // size of results, determined by function and input column data type int32_t interBufBytes; // internal buffer size bool hasNull; // null value exist in current block - bool requireNull; // require null in some function + bool requireNull; // require null in some function bool stableQuery; - int16_t functionId; // function id - void * aInputElemBuf; - char * aOutputBuf; // final result output buffer, point to sdata->data - uint8_t currentStage; // record current running step, default: 0 - int64_t nStartQueryTimestamp; // timestamp range of current query when function is executed on a specific data block + int16_t functionId; // function id + char * pOutput; // final result output buffer, point to sdata->data + uint8_t currentStage; // record current running step, default: 0 + int64_t startTs; // timestamp range of current query when function is executed on a specific data block int32_t numOfParams; - tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param */ + tVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param int64_t *ptsList; // corresponding timestamp array list void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; @@ -202,17 +199,16 @@ typedef struct SQLFunctionCtx { SPoint1 end; } SQLFunctionCtx; -typedef struct SQLAggFuncElem { - char aName[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; - - uint8_t nAggIdx; // index of function in aAggs +typedef struct SAggFunctionInfo { + char name[TSDB_FUNCTIONS_NAME_MAX_LENGTH]; + uint8_t index; // index of function in aAggs int8_t stableFuncId; // transfer function for super table query - uint16_t nStatus; + uint16_t status; bool (*init)(SQLFunctionCtx *pCtx); // setup the execute environment void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function - void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version + void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version, todo merge with blockwise function // some sql function require scan data twice or more, e.g.,stddev, percentile void (*xNextStep)(SQLFunctionCtx *pCtx); @@ -221,20 +217,21 @@ typedef struct SQLAggFuncElem { void (*xFinalize)(SQLFunctionCtx *pCtx); void (*mergeFunc)(SQLFunctionCtx *pCtx); - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId); -} SQLAggFuncElem; + int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); +} SAggFunctionInfo; #define GET_RES_INFO(ctx) ((ctx)->resultInfo) int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable); +int32_t isValidFunction(const char* name, int32_t len); #define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0) #define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0) #define IS_SINGLEOUTPUT(x) (((x)&TSDB_FUNCSTATE_SO) != 0) #define IS_OUTER_FORWARD(x) (((x)&TSDB_FUNCSTATE_OF) != 0) -/* determine the real data need to calculated the result */ +// determine the real data need to calculated the result enum { BLK_DATA_NO_NEEDED = 0x0, BLK_DATA_STATIS_NEEDED = 0x1, @@ -249,12 +246,16 @@ typedef struct STwaInfo { STimeWindow win; } STwaInfo; +struct SBufferWriter; +void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw); +void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist); + /* global sql function array */ -extern struct SQLAggFuncElem aAggs[]; +extern struct SAggFunctionInfo aAggs[]; extern int32_t functionCompatList[]; // compatible check array list -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval); +bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval); /** * the numOfRes should be kept, since it may be used later @@ -265,18 +266,18 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha (_r)->initialized = false; \ } while (0) -static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) { +static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; pResInfo->hasResult = false; pResInfo->numOfRes = 0; - memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen); + memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen); } #ifdef __cplusplus } #endif -#endif // TDENGINE_TSQLFUNCTION_H +#endif // TDENGINE_QAGGMAIN_H diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 79d98432c87ea7f06187d39f258fcb00d73f30a6..5ff574ec673119d3837ef57d119c778e6cd8c285 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -12,12 +12,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_QUERYEXECUTOR_H -#define TDENGINE_QUERYEXECUTOR_H +#ifndef TDENGINE_QEXECUTOR_H +#define TDENGINE_QEXECUTOR_H #include "os.h" #include "hash.h" +#include "qAggMain.h" #include "qFill.h" #include "qResultbuf.h" #include "qSqlparser.h" @@ -27,12 +28,35 @@ #include "tarray.h" #include "tlockfree.h" #include "tsdb.h" -#include "tsqlfunction.h" struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) +#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) +#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) + +#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) + +#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) + +enum { + // when query starts to execute, this status will set + QUERY_NOT_COMPLETED = 0x1u, + + /* query is over + * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. + * 2. when all data within queried time window, it is also denoted as query_completed + */ + QUERY_COMPLETED = 0x2u, + + /* when the result is not completed return to client, this status will be + * usually used in case of interval query with interpolation option + */ + QUERY_OVER = 0x4u, +}; + typedef struct SResultRowPool { int32_t elemSize; int32_t blockSize; @@ -56,17 +80,18 @@ typedef struct SSqlGroupbyExpr { typedef struct SResultRow { int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer - int32_t rowId:29; // row index in buffer page + int32_t offset:29; // row index in buffer page bool startInterp; // the time window start timestamp has done the interpolation already. bool endInterp; // the time window end timestamp has done the interpolation already. bool closed; // this result status: closed or opened uint32_t numOfRows; // number of rows of current time window SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo - union {STimeWindow win; char* key;}; // start key of current time window + union {STimeWindow win; char* key;}; // start key of current result row } SResultRow; typedef struct SGroupResInfo { - int32_t rowId; + int32_t totalGroup; + int32_t currentGroup; int32_t index; SArray* pRows; // SArray } SGroupResInfo; @@ -75,12 +100,11 @@ typedef struct SGroupResInfo { * If the number of generated results is greater than this value, * query query will be halt and return results to client immediate. */ -typedef struct SResultRec { +typedef struct SRspResultInfo { int64_t total; // total generated result size in rows - int64_t rows; // current result set size in rows - int64_t capacity; // capacity of current result output buffer + int32_t capacity; // capacity of current result output buffer int32_t threshold; // result size threshold in rows. -} SResultRec; +} SRspResultInfo; typedef struct SResultRowInfo { SResultRow** pResult; // result list @@ -107,12 +131,11 @@ typedef struct SSingleColumnFilterInfo { typedef struct STableQueryInfo { TSKEY lastKey; int32_t groupIndex; // group id in table list - int16_t queryRangeSet; // denote if the query range is set, only available for interval query tVariant tag; STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list - SResultRowInfo windowResInfo; + SResultRowInfo resInfo; } STableQueryInfo; typedef struct SQueryCostInfo { @@ -143,71 +166,141 @@ typedef struct { int64_t ts; } SOrderedPrjQueryInfo; +typedef struct { + char* tags; + SArray* pResult; // SArray +} SInterResult; + +typedef struct SSDataBlock { + SDataStatis *pBlockStatis; + SArray *pDataBlock; + SDataBlockInfo info; +} SSDataBlock; + typedef struct SQuery { + SLimitVal limit; + + bool stableQuery; // super table query or not + bool topBotQuery; // TODO used bitwise flag + bool groupbyColumn; // denote if this is a groupby normal column query + bool hasTagResults; // if there are tag values in final result or not + bool timeWindowInterpo;// if the time window start/end required interpolation + bool queryBlockDist; // if query data block distribution + bool stabledev; // super table stddev query + int32_t interBufSize; // intermediate buffer sizse + + SOrderVal order; int16_t numOfCols; int16_t numOfTags; - SOrderVal order; + STimeWindow window; SInterval interval; + SSessionWindow sw; int16_t precision; int16_t numOfOutput; int16_t fillType; - int16_t checkResultBuf; // check if the buffer is full during scan each block - SLimitVal limit; - int32_t rowSize; + int16_t checkResultBuf; // check if the buffer is full during scan each block + + int32_t srcRowSize; // todo extract struct + int32_t resultRowSize; + int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. + int32_t maxSrcColumnSize; + int32_t tagLen; // tag value length of current query SSqlGroupbyExpr* pGroupbyExpr; SExprInfo* pExpr1; SExprInfo* pExpr2; int32_t numOfExpr2; - SColumnInfo* colList; SColumnInfo* tagColList; int32_t numOfFilterCols; int64_t* fillVal; - uint32_t status; // query status - SResultRec rec; - int32_t pos; - tFilePage** sdata; - STableQueryInfo* current; - - SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. + SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. SSingleColumnFilterInfo* pFilterInfo; + + STableQueryInfo* current; + void* tsdb; + SMemRef memRef; + STableGroupInfo tableGroupInfo; // table list SArray + int32_t vgId; } SQuery; +typedef SSDataBlock* (*__operator_fn_t)(void* param); +typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); + +struct SOperatorInfo; + typedef struct SQueryRuntimeEnv { - jmp_buf env; - SQuery* pQuery; - SQLFunctionCtx* pCtx; - int32_t numOfRowsPerPage; - uint16_t* offset; - uint16_t scanFlag; // denotes reversed scan of data or not - SFillInfo* pFillInfo; - SResultRowInfo windowResInfo; - STSBuf* pTsBuf; - STSCursor cur; - SQueryCostInfo summary; - void* pQueryHandle; - void* pSecQueryHandle; // another thread for - bool stableQuery; // super table query or not - bool topBotQuery; // TODO used bitwise flag - bool groupbyColumn; // denote if this is a groupby normal column query - bool hasTagResults; // if there are tag values in final result or not - bool timeWindowInterpo;// if the time window start/end required interpolation - bool queryWindowIdentical; // all query time windows are identical for all tables in one group - int32_t interBufSize; // intermediate buffer sizse - int32_t prevGroupId; // previous executed group id - SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file - SHashObj* pResultRowHashTable; // quick locate the window object for each result - char* keyBuf; // window key buffer - SResultRowPool* pool; // window result object pool - - int32_t* rowCellInfoOffset;// offset value for each row result cell info - char** prevRow; - char** nextRow; - - SArithmeticSupport *sasArray; + jmp_buf env; + SQuery* pQuery; + uint32_t status; // query status + void* qinfo; + uint8_t scanFlag; // denotes reversed scan of data or not + void* pQueryHandle; + + int32_t prevGroupId; // previous executed group id + SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + char* keyBuf; // window key buffer + SResultRowPool* pool; // window result object pool + char** prevRow; + + SArray* prevResult; // intermediate result, SArray + STSBuf* pTsBuf; // timestamp filter list + STSCursor cur; + + char* tagVal; // tag value of current data block + SArithmeticSupport *sasArray; + + SSDataBlock *outputBuf; + STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure + struct SOperatorInfo *proot; + struct SOperatorInfo *pTableScanner; // table scan operator + SGroupResInfo groupResInfo; + int64_t currentOffset; // dynamic offset value + + SRspResultInfo resultInfo; + SHashObj *pTableRetrieveTsMap; } SQueryRuntimeEnv; +enum { + OP_IN_EXECUTING = 1, + OP_RES_TO_RETURN = 2, + OP_EXEC_DONE = 3, +}; + +enum OPERATOR_TYPE_E { + OP_TableScan = 1, + OP_DataBlocksOptScan = 2, + OP_TableSeqScan = 3, + OP_TagScan = 4, + OP_TableBlockInfoScan= 5, + OP_Aggregate = 6, + OP_Arithmetic = 7, + OP_Groupby = 8, + OP_Limit = 9, + OP_Offset = 10, + OP_TimeWindow = 11, + OP_SessionWindow = 12, + OP_Fill = 13, + OP_MultiTableAggregate = 14, + OP_MultiTableTimeInterval = 15, +}; + +typedef struct SOperatorInfo { + uint8_t operatorType; + bool blockingOptr; // block operator or not + uint8_t status; // denote if current operator is completed + int32_t numOfOutput; // number of columns of the current operator results + char *name; // name, used to show the query execution plan + void *info; // extension attribution + SExprInfo *pExpr; + SQueryRuntimeEnv *pRuntimeEnv; + + struct SOperatorInfo *upstream; + __operator_fn_t exec; + __optr_cleanup_fn_t cleanup; +} SOperatorInfo; + enum { QUERY_RESULT_NOT_READY = 1, QUERY_RESULT_READY = 2, @@ -215,23 +308,12 @@ enum { typedef struct SQInfo { void* signature; - int32_t code; // error code to returned to client - int64_t owner; // if it is in execution - void* tsdb; - SMemRef memRef; - int32_t vgId; - STableGroupInfo tableGroupInfo; // table list SArray - STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure - SQueryRuntimeEnv runtimeEnv; - SHashObj* arrTableIdInfo; - int32_t groupIndex; + uint64_t qId; + int32_t code; // error code to returned to client + int64_t owner; // if it is in execution - /* - * the query is executed position on which meter of the whole list. - * when the index reaches the last one of the list, it means the query is completed. - */ - int32_t tableIndex; - SGroupResInfo groupResInfo; + SQueryRuntimeEnv runtimeEnv; + SQuery query; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads @@ -240,6 +322,133 @@ typedef struct SQInfo { void* rspContext; // response context int64_t startExecTs; // start to exec timestamp char* sql; // query sql string + SQueryCostInfo summary; } SQInfo; -#endif // TDENGINE_QUERYEXECUTOR_H +typedef struct SQueryParam { + char *sql; + char *tagCond; + char *tbnameCond; + char *prevResult; + SArray *pTableIdList; + SSqlFuncMsg **pExprMsg; + SSqlFuncMsg **pSecExprMsg; + SExprInfo *pExprs; + SExprInfo *pSecExprs; + + SColIndex *pGroupColIndex; + SColumnInfo *pTagColumnInfo; + SSqlGroupbyExpr *pGroupbyExpr; +} SQueryParam; + +typedef struct STableScanInfo { + void *pQueryHandle; + int32_t numOfBlocks; + int32_t numOfSkipped; + int32_t numOfBlockStatis; + int64_t numOfRows; + + int32_t order; // scan order + int32_t times; // repeat counts + int32_t current; + int32_t reverseTimes; // 0 by default + + SQLFunctionCtx *pCtx; // next operator query context + SResultRowInfo *pResultRowInfo; + int32_t *rowCellInfoOffset; + SExprInfo *pExpr; + SSDataBlock block; + bool loadExternalRows; // load external rows (prev & next rows) + int32_t numOfOutput; + int64_t elapsedTime; + + int32_t tableIndex; +} STableScanInfo; + +typedef struct STagScanInfo { + SColumnInfo* pCols; + SSDataBlock* pRes; + int32_t totalTables; + int32_t currentIndex; +} STagScanInfo; + +typedef struct SOptrBasicInfo { + SResultRowInfo resultRowInfo; + int32_t *rowCellInfoOffset; // offset value for each row result cell info + SQLFunctionCtx *pCtx; + SSDataBlock *pRes; +} SOptrBasicInfo; + +typedef struct SOptrBasicInfo STableIntervalOperatorInfo; + +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + uint32_t seed; +} SAggOperatorInfo; + +typedef struct SArithOperatorInfo { + SOptrBasicInfo binfo; + int32_t bufCapacity; + uint32_t seed; +} SArithOperatorInfo; + +typedef struct SLimitOperatorInfo { + int64_t limit; + int64_t total; +} SLimitOperatorInfo; + +typedef struct SOffsetOperatorInfo { + int64_t offset; +} SOffsetOperatorInfo; + +typedef struct SFillOperatorInfo { + SFillInfo *pFillInfo; + SSDataBlock *pRes; + int64_t totalInputRows; +} SFillOperatorInfo; + +typedef struct SGroupbyOperatorInfo { + SOptrBasicInfo binfo; + int32_t colIndex; + char *prevData; // previous group by value +} SGroupbyOperatorInfo; + +typedef struct SSWindowOperatorInfo { + SOptrBasicInfo binfo; + STimeWindow curWindow; // current time window + TSKEY prevTs; // previous timestamp + int32_t numOfRows; // number of rows + int32_t start; // start row index +} SSWindowOperatorInfo; + +void freeParam(SQueryParam *param); +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param); +int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg, + SColumnInfo* pTagCols); +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, + SSqlFuncMsg **pExprMsg, SExprInfo *prevExpr); + +SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); +SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql, uint64_t *qId); +int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, SQueryParam* param, bool isSTable); +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); + +bool isQueryKilled(SQInfo *pQInfo); +int32_t checkForQueryBuf(size_t numOfTables); +bool doBuildResCheck(SQInfo* pQInfo); +void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status); + +bool onlyQueryTags(SQuery* pQuery); +bool isValidQInfo(void *param); + +int32_t doDumpQueryResult(SQInfo *pQInfo, char *data); + +size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); +void setQueryKilled(SQInfo *pQInfo); +void queryCostStatis(SQInfo *pQInfo); +void freeQInfo(SQInfo *pQInfo); + +int32_t getMaximumIdleDurationSec(); + +#endif // TDENGINE_QEXECUTOR_H diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index 385ae8854327b036cbb6ee46de70e97ae56ce25b..00ac86caf4a079e01ad239496370d97ac28e84f2 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -24,6 +24,8 @@ extern "C" { #include "qExtbuffer.h" #include "taosdef.h" +struct SSDataBlock; + typedef struct { STColumn col; // column info int16_t functionId; // sql function id @@ -68,7 +70,7 @@ typedef struct SPoint { void * val; } SPoint; -SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, +SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, SFillColInfo* pFillCol, void* handle); @@ -78,17 +80,17 @@ void* taosDestroyFillInfo(SFillInfo *pFillInfo); void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); -void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput); +void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput); void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput); -int64_t getNumOfResWithFill(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); +bool taosFillHasMoreResults(SFillInfo* pFillInfo); -int32_t taosNumOfRemainRows(SFillInfo *pFillInfo); +int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows); -int32_t taosGetLinearInterpolationVal(int32_t type, SPoint *point1, SPoint *point2, SPoint *point); +int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity); +int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity); #ifdef __cplusplus } diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index 704df9f3f29cc0653c353ed194e02e72cb4c5fb2..ef43a9621a8298fff857a252a1b67a86988d786e 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -55,7 +55,6 @@ typedef struct SResultBufStatis { } SResultBufStatis; typedef struct SDiskbasedResultBuf { - int32_t numOfRowsPerPage; int32_t numOfPages; int64_t totalBufSize; int64_t fileSize; // disk file size @@ -73,11 +72,11 @@ typedef struct SDiskbasedResultBuf { bool comp; // compressed before flushed to disk int32_t nextPos; // next page flush position - const void* handle; // for debug purpose + uint64_t qId; // for debug purpose SResultBufStatis statis; } SDiskbasedResultBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (256L) // in bytes +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes #define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1} /** @@ -89,8 +88,7 @@ typedef struct SDiskbasedResultBuf { * @param handle * @return */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize, - int32_t inMemBufSize, const void* handle); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId); /** * @@ -101,13 +99,6 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro */ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); -/** - * - * @param pResultBuf - * @return - */ -size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf); - /** * * @param pResultBuf diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 56e676ef167b8c9f1a694ef0571d1b0318ef6992..3ce81787f05b7336d25df39fbd5cad75685e1527 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -27,6 +27,23 @@ extern "C" { #include "tvariant.h" #define ParseTOKENTYPE SStrToken + +#define NON_ARITHMEIC_EXPR 0 +#define NORMAL_ARITHMETIC 1 +#define AGG_ARIGHTMEIC 2 + +enum SQL_NODE_TYPE { + SQL_NODE_TABLE_COLUMN= 1, + SQL_NODE_SQLFUNCTION = 2, + SQL_NODE_VALUE = 3, + SQL_NODE_EXPR = 4, +}; + +enum SQL_NODE_FROM_TYPE { + SQL_NODE_FROM_SUBQUERY = 1, + SQL_NODE_FROM_NAMELIST = 2, +}; + extern char tTokenTypeSwitcher[13]; #define toTSDBType(x) \ @@ -38,226 +55,235 @@ extern char tTokenTypeSwitcher[13]; } \ } while (0) +#define TPARSER_HAS_TOKEN(_t) ((_t).n > 0) +#define TPARSER_SET_NONE_TOKEN(_t) ((_t).n = 0) + typedef struct SLimitVal { - int64_t limit; - int64_t offset; + int64_t limit; + int64_t offset; } SLimitVal; typedef struct SOrderVal { - uint32_t order; - int32_t orderColId; + uint32_t order; + int32_t orderColId; } SOrderVal; typedef struct tVariantListItem { - tVariant pVar; - uint8_t sortOrder; + tVariant pVar; + uint8_t sortOrder; } tVariantListItem; typedef struct SIntervalVal { - SStrToken interval; - SStrToken offset; + SStrToken interval; + SStrToken offset; } SIntervalVal; -typedef struct SQuerySQL { - struct tSQLExprList *pSelection; // select clause - SArray * from; // from clause SArray - struct tSQLExpr * pWhere; // where clause [optional] - SArray * pGroupby; // groupby clause, only for tags[optional], SArray - SArray * pSortOrder; // orderby [optional], SArray - SStrToken interval; // interval [optional] - SStrToken offset; // offset window [optional] - SStrToken sliding; // sliding window [optional] - SLimitVal limit; // limit offset [optional] - SLimitVal slimit; // group limit offset [optional] - SArray * fillType; // fill type[optional], SArray - SStrToken selectToken; // sql string -} SQuerySQL; +typedef struct SSessionWindowVal { + SStrToken col; + SStrToken gap; +} SSessionWindowVal; + +struct SFromInfo; + +typedef struct SQuerySqlNode { + struct SArray *pSelectList; // select clause + struct SFromInfo *from; // from clause SArray + struct tSqlExpr *pWhere; // where clause [optional] + SArray *pGroupby; // groupby clause, only for tags[optional], SArray + SArray *pSortOrder; // orderby [optional], SArray + SArray *fillType; // fill type[optional], SArray + SIntervalVal interval; // (interval, interval_offset) [optional] + SSessionWindowVal sessionVal; // session window [optional] + SStrToken sliding; // sliding window [optional] + SLimitVal limit; // limit offset [optional] + SLimitVal slimit; // group limit offset [optional] + SStrToken sqlstr; // sql string in select clause +} SQuerySqlNode; + +typedef struct STableNamePair { + SStrToken name; + SStrToken aliasName; +} STableNamePair; + +typedef struct SSubclauseInfo { // "UNION" multiple select sub-clause + SQuerySqlNode **pClause; + int32_t numOfClause; +} SSubclauseInfo; + +typedef struct SFromInfo { + int32_t type; // nested query|table name list + union { + SSubclauseInfo *pNode; + SArray *tableList; // SArray + }; +} SFromInfo; typedef struct SCreatedTableInfo { - SStrToken name; // table name token - SStrToken stableName; // super table name token , for using clause - SArray *pTagVals; // create by using super table, tag value - char *fullname; // table full name - STagData tagdata; // true tag data, super table full name is in STagData - int8_t igExist; // ignore if exists + SStrToken name; // table name token + SStrToken stableName; // super table name token , for using clause + SArray *pTagNames; // create by using super table, tag name + SArray *pTagVals; // create by using super table, tag value + char *fullname; // table full name + STagData tagdata; // true tag data, super table full name is in STagData + int8_t igExist; // ignore if exists } SCreatedTableInfo; -typedef struct SCreateTableSQL { - SStrToken name; // table name, create table [name] xxx - int8_t type; // create normal table/from super table/ stream - bool existCheck; +typedef struct SCreateTableSql { + SStrToken name; // table name, create table [name] xxx + int8_t type; // create normal table/from super table/ stream + bool existCheck; struct { - SArray *pTagColumns; // SArray - SArray *pColumns; // SArray + SArray *pTagColumns; // SArray + SArray *pColumns; // SArray } colInfo; - SArray *childTableInfo; // SArray - SQuerySQL *pSelect; -} SCreateTableSQL; - -typedef struct SAlterTableSQL { - SStrToken name; - int16_t type; - STagData tagData; - SArray *pAddColumns; // SArray - SArray *varList; // set t=val or: change src dst, SArray -} SAlterTableSQL; - -typedef struct SCreateDBInfo { - SStrToken dbname; - int32_t replica; - int32_t cacheBlockSize; - int32_t maxTablesPerVnode; - int32_t numOfBlocks; - int32_t daysPerFile; - int32_t minRowsPerBlock; - int32_t maxRowsPerBlock; - int32_t fsyncPeriod; - int64_t commitTime; - int32_t walLevel; - int32_t quorum; - int32_t compressionLevel; - SStrToken precision; - bool ignoreExists; - int8_t update; - int8_t cachelast; - - SArray *keep; -} SCreateDBInfo; - -typedef struct SCreateAcctSQL { - int32_t maxUsers; - int32_t maxDbs; - int32_t maxTimeSeries; - int32_t maxStreams; - int32_t maxPointsPerSecond; - int64_t maxStorage; - int64_t maxQueryTime; - int32_t maxConnections; - SStrToken stat; -} SCreateAcctSQL; + SArray *childTableInfo; // SArray + SQuerySqlNode *pSelect; +} SCreateTableSql; + +typedef struct SAlterTableInfo { + SStrToken name; + int16_t tableType; + int16_t type; + STagData tagData; + SArray *pAddColumns; // SArray + SArray *varList; // set t=val or: change src dst, SArray +} SAlterTableInfo; + +typedef struct SCreateDbInfo { + SStrToken dbname; + int32_t replica; + int32_t cacheBlockSize; + int32_t maxTablesPerVnode; + int32_t numOfBlocks; + int32_t daysPerFile; + int32_t minRowsPerBlock; + int32_t maxRowsPerBlock; + int32_t fsyncPeriod; + int64_t commitTime; + int32_t walLevel; + int32_t quorum; + int32_t compressionLevel; + SStrToken precision; + bool ignoreExists; + int8_t update; + int8_t cachelast; + SArray *keep; + int8_t dbType; + int16_t partitions; +} SCreateDbInfo; + +typedef struct SCreateAcctInfo { + int32_t maxUsers; + int32_t maxDbs; + int32_t maxTimeSeries; + int32_t maxStreams; + int32_t maxPointsPerSecond; + int64_t maxStorage; + int64_t maxQueryTime; + int32_t maxConnections; + SStrToken stat; +} SCreateAcctInfo; typedef struct SShowInfo { - uint8_t showType; - SStrToken prefix; - SStrToken pattern; + uint8_t showType; + SStrToken prefix; + SStrToken pattern; } SShowInfo; typedef struct SUserInfo { - SStrToken user; - SStrToken passwd; - SStrToken privilege; - int16_t type; + SStrToken user; + SStrToken passwd; + SStrToken privilege; + int16_t type; } SUserInfo; -typedef struct tDCLSQL { - int32_t nTokens; /* Number of expressions on the list */ - int32_t nAlloc; /* Number of entries allocated below */ - SStrToken *a; /* one entry for element */ - bool existsCheck; - +typedef struct SMiscInfo { + SArray *a; // SArray + bool existsCheck; + int16_t dbType; + int16_t tableType; + SUserInfo user; union { - SCreateDBInfo dbOpt; - SCreateAcctSQL acctOpt; - SShowInfo showOpt; - SStrToken ip; + SCreateDbInfo dbOpt; + SCreateAcctInfo acctOpt; + SShowInfo showOpt; + SStrToken id; }; - - SUserInfo user; - -} tDCLSQL; - -typedef struct SSubclauseInfo { // "UNION" multiple select sub-clause - SQuerySQL **pClause; - int32_t numOfClause; -} SSubclauseInfo; +} SMiscInfo; typedef struct SSqlInfo { int32_t type; bool valid; - + SSubclauseInfo subclauseInfo; + char msg[256]; union { - SCreateTableSQL *pCreateTableInfo; - SAlterTableSQL *pAlterInfo; - tDCLSQL *pDCLInfo; + SCreateTableSql *pCreateTableInfo; + SAlterTableInfo *pAlterInfo; + SMiscInfo *pMiscInfo; }; - - SSubclauseInfo subclauseInfo; - char pzErrMsg[256]; } SSqlInfo; -typedef struct tSQLExpr { - uint32_t nSQLOptr; // TK_FUNCTION: sql function, TK_LE: less than(binary expr) - - // the full sql string of function(col, param), which is actually the raw - // field name, since the function name is kept in nSQLOptr already - SStrToken operand; - SStrToken colInfo; // field id - tVariant val; // value only for string, float, int - SStrToken token; // original sql expr string - - struct tSQLExpr *pLeft; // left child - struct tSQLExpr *pRight; // right child - struct tSQLExprList *pParam; // function parameters -} tSQLExpr; - -// used in select clause. select from xxx -typedef struct tSqlExprItem { - tSQLExpr *pNode; // The list of expressions - char * aliasName; // alias name, null-terminated string -} tSqlExprItem; +typedef struct tSqlExpr { + uint16_t type; // sql node type + uint32_t tokenId; // TK_LE: less than(binary expr) -// todo refactor by using SArray -typedef struct tSQLExprList { - int32_t nExpr; /* Number of expressions on the list */ - int32_t nAlloc; /* Number of entries allocated below */ - tSqlExprItem *a; /* One entry for each expression */ -} tSQLExprList; + // the whole string of the function(col, param), while the function name is kept in token + SStrToken operand; + uint32_t functionId; // function id -/** - * - * @param yyp The parser - * @param yymajor The major token code number - * @param yyminor The value for the token - */ -void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); + SStrToken colInfo; // table column info + tVariant value; // the use input value + SStrToken token; // original sql expr string -/** - * - * @param p The parser to be deleted - * @param freeProc Function used to reclaim memory - */ -void ParseFree(void *p, void (*freeProc)(void *)); + struct tSqlExpr *pLeft; // left child + struct tSqlExpr *pRight; // right child + struct SArray *pParam; // function parameters list +} tSqlExpr; + +// used in select clause. select from xxx +typedef struct tSqlExprItem { + tSqlExpr *pNode; // The list of expressions + char *aliasName; // alias name, null-terminated string + bool distinct; +} tSqlExprItem; SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder); SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder); -tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType); +SFromInfo *setTableNameList(SFromInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias); +SFromInfo *setSubquery(SFromInfo* pFromInfo, SQuerySqlNode *pSqlNode); +void *destroyFromInfo(SFromInfo* pFromInfo); -void tSqlExprDestroy(tSQLExpr *pExpr); +// sql expr leaf node +tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType); +tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); +tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType); +tSqlExpr *tSqlExprClone(tSqlExpr *pSrc); +void tSqlExprCompact(tSqlExpr** pExpr); +bool tSqlExprIsLeaf(tSqlExpr* pExpr); +bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr); +void tSqlExprDestroy(tSqlExpr *pExpr); +SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct, SStrToken *pToken); +void tSqlExprListDestroy(SArray *pList); -void tSqlExprListDestroy(tSQLExprList *pList); +SQuerySqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelectList, SFromInfo *pFrom, tSqlExpr *pWhere, + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps, + SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pgLimit); -SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, - SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); +SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SQuerySqlNode *pSelect, int32_t type); -SCreateTableSQL *tSetCreateSqlElems(SArray *pCols, SArray *pTags, SQuerySQL *pSelect, int32_t type); - -void tSqlExprNodeDestroy(tSQLExpr *pExpr); - -SAlterTableSQL * tAlterTableSqlElems(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type); -SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists); +SAlterTableInfo *tSetAlterTableInfo(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableTable); +SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists); void destroyAllSelectClause(SSubclauseInfo *pSql); -void doDestroyQuerySql(SQuerySQL *pSql); +void destroyQuerySqlNode(SQuerySqlNode *pSql); void freeCreateTableInfo(void* p); -SSqlInfo * setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type); +SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type); SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); @@ -266,45 +292,54 @@ void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken void SqlInfoDestroy(SSqlInfo *pInfo); -void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); +void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck,int16_t dbType,int16_t tableType); void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken); +void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists); -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists); - -void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo); void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip); void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); -void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo); +void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo); +void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo); // prefix show db.tables; -void setDbName(SStrToken *pCpxName, SStrToken *pDb); - -tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType); +void tSetDbName(SStrToken *pCpxName, SStrToken *pDb); -tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); +void tSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); +void tSetColumnType(TAOS_FIELD *pField, SStrToken *type); -void tSqlSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); +/** + * + * @param yyp The parser + * @param yymajor The major token code number + * @param yyminor The value for the token + */ +void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); -void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type); +/** + * + * @param p The parser to be deleted + * @param freeProc Function used to reclaim memory + */ +void ParseFree(void *p, void (*freeProc)(void *)); +/** + * + * @param mallocProc The parser allocator + * @return + */ void *ParseAlloc(void *(*mallocProc)(size_t)); -enum { - TSQL_NODE_TYPE_EXPR = 0x1, - TSQL_NODE_TYPE_ID = 0x2, - TSQL_NODE_TYPE_VALUE = 0x4, -}; - -#define NON_ARITHMEIC_EXPR 0 -#define NORMAL_ARITHMETIC 1 -#define AGG_ARIGHTMEIC 2 - -SSqlInfo qSQLParse(const char *str); +/** + * + * @param str sql string + * @return sql ast + */ +SSqlInfo qSqlParse(const char *str); #ifdef __cplusplus } diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 5d055782c9b82a1444c97a62d429cc2ba9a53986..00cc4e897f130348b81a7d96419c1b292cacca8c 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -112,13 +112,11 @@ STSBuf* tsBufClone(STSBuf* pTSBuf); STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); -void tsBufFlush(STSBuf* pTSBuf); - +void tsBufFlush(STSBuf* pTSBuf); void tsBufResetPos(STSBuf* pTSBuf); -STSElem tsBufGetElem(STSBuf* pTSBuf); - bool tsBufNextPos(STSBuf* pTSBuf); +STSElem tsBufGetElem(STSBuf* pTSBuf); STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 27bf7f2d96983567fb0f041545a09c1cb4fe24d8..cdd8b0707a86404597fc1379e0743173734a1db1 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -15,6 +15,8 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#include "tbuffer.h" + #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ do { \ assert(sizeof(_uid) == sizeof(uint64_t)); \ @@ -23,9 +25,10 @@ } while (0) #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) +#define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) -#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1) +#define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!(sq)))? (_q)->pExpr1[1].base.arg->argValue.i64:1) int32_t getOutputInterResultBufSize(SQuery* pQuery); @@ -42,22 +45,18 @@ void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); -SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); +SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { assert(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); return pResultRowInfo->pResult[slot]; } -static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult, - tFilePage* page) { - assert(pResult != NULL && pRuntimeEnv != NULL); - - SQuery *pQuery = pRuntimeEnv->pQuery; +static FORCE_INLINE char *getPosInResultPage(SQuery *pQuery, tFilePage* page, int32_t rowOffset, int16_t offset) { + assert(rowOffset >= 0 && pQuery != NULL); - int32_t realRowId = (int32_t)(pResult->rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); - return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + - pQuery->pExpr1[columnIndex].bytes * realRowId; + int32_t numOfRows = (int32_t)GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery); + return ((char *)page->data) + rowOffset + offset * numOfRows; } bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); @@ -72,7 +71,23 @@ void* destroyResultRowPool(SResultRowPool* p); int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); -bool isPointInterpoQuery(SQuery *pQuery); +typedef struct { + SArray* pResult; // SArray + int32_t colId; +} SStddevInterResult; + +void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen); +SArray* interResFromBinary(const char* data, int32_t len); +void freeInterResult(void* param); + +void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo); +void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); +bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo); +bool hasRemainData(SGroupResInfo* pGroupResInfo); + +bool incNextGroup(SGroupResInfo* pGroupResInfo); +int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset); #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/queryLog.h b/src/query/inc/queryLog.h index 26544ab0f9152a9c0282893f20956e99a1987afe..5c48c43c45fdb5cffdcc01901f1ede4a3b98d240 100644 --- a/src/query/inc/queryLog.h +++ b/src/query/inc/queryLog.h @@ -30,6 +30,7 @@ extern uint32_t qDebugFlag; #define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", 255, __VA_ARGS__); }} while(0) #define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDump(a, l) do { if (qDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)a, l); }} while(0) #ifdef __cplusplus } diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 1fa1369bb5da5bfe0fd4a08ce7c059396b8fa394..174b31a9c6cec7b4c3274b35939fb3773cc6d70d 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -36,7 +36,7 @@ %syntax_error { pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->pzErrMsg); + int32_t outputBufLen = tListLen(pInfo->msg); int32_t len = 0; if(TOKEN.z) { @@ -46,13 +46,13 @@ if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { char tmpstr[128] = {0}; memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->pzErrMsg, msg, tmpstr); + len = sprintf(pInfo->msg, msg, tmpstr); } else { - len = sprintf(pInfo->pzErrMsg, msg, &TOKEN.z[0]); + len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); } } else { - len = sprintf(pInfo->pzErrMsg, "Incomplete SQL statement"); + len = sprintf(pInfo->msg, "Incomplete SQL statement"); } assert(len <= outputBufLen); @@ -64,6 +64,7 @@ program ::= cmd. {} //////////////////////////////////THE SHOW STATEMENT/////////////////////////////////////////// cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} +cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} @@ -91,11 +92,11 @@ cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } cmd ::= SHOW CREATE TABLE ids(X) cpxName(Y). { X.n += Y.n; - setDCLSQLElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &X); + setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &X); } cmd ::= SHOW CREATE DATABASE ids(X). { - setDCLSQLElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &X); + setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &X); } cmd ::= SHOW dbPrefix(X) TABLES. { @@ -112,50 +113,59 @@ cmd ::= SHOW dbPrefix(X) STABLES. { cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { SStrToken token; - setDbName(&token, &X); + tSetDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { SStrToken token; - setDbName(&token, &X); + tSetDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { SStrToken token; - setDbName(&token, &X); + tSetDbName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } //drop configure for tables cmd ::= DROP TABLE ifexists(Y) ids(X) cpxName(Z). { X.n += Z.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y); + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, -1); } -cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y); } -cmd ::= DROP DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } -cmd ::= DROP USER ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } -cmd ::= DROP ACCOUNT ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } +//drop stable +cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). { + X.n += Z.n; + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &X, &Y, -1, TSDB_SUPER_TABLE); +} + +cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); } +cmd ::= DROP TOPIC ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); } + +cmd ::= DROP DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); } +cmd ::= DROP USER ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); } +cmd ::= DROP ACCOUNT ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &X); } /////////////////////////////////THE USE STATEMENT////////////////////////////////////////// -cmd ::= USE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_USE_DB, 1, &X);} +cmd ::= USE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &X);} /////////////////////////////////THE DESCRIBE STATEMENT///////////////////////////////////// cmd ::= DESCRIBE ids(X) cpxName(Y). { X.n += Y.n; - setDCLSQLElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); + setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &X); } /////////////////////////////////THE ALTER STATEMENT//////////////////////////////////////// cmd ::= ALTER USER ids(X) PASS ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &X, &Y, NULL); } cmd ::= ALTER USER ids(X) PRIVILEGE ids(Y). { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &X, NULL, &Y);} -cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } -cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } -cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } -cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} +cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &X, &Y); } +cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } +cmd ::= ALTER LOCAL ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } +cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } +cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} +cmd ::= ALTER TOPIC ids(X) alter_topic_optr(Y). { SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} @@ -177,10 +187,11 @@ ifnotexists(X) ::= . { X.n = 0;} /////////////////////////////////THE CREATE STATEMENT/////////////////////////////////////// //create option for dnode/db/user/account -cmd ::= CREATE DNODE ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} +cmd ::= CREATE DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &X);} cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);} -cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} +cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} +cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);} cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);} pps(Y) ::= . { Y.n = 0; } @@ -210,7 +221,7 @@ conns(Y) ::= CONNS INTEGER(X). { Y = X; } state(Y) ::= . { Y.n = 0; } state(Y) ::= STATE ids(X). { Y = X; } -%type acct_optr {SCreateAcctSQL} +%type acct_optr {SCreateAcctInfo} acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K) conns(L) state(M). { Y.maxUsers = (K.n>0)?atoi(K.z):-1; Y.maxDbs = (E.n>0)?atoi(E.z):-1; @@ -241,9 +252,10 @@ comp(Y) ::= COMP INTEGER(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; } update(Y) ::= UPDATE INTEGER(X). { Y = X; } cachelast(Y) ::= CACHELAST INTEGER(X). { Y = X; } +partitions(Y) ::= PARTITIONS INTEGER(X). { Y = X; } -%type db_optr {SCreateDBInfo} -db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} +%type db_optr {SCreateDbInfo} +db_optr(Y) ::= . {setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;} db_optr(Y) ::= db_optr(Z) cache(X). { Y = Z; Y.cacheBlockSize = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } @@ -261,8 +273,13 @@ db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } -%type alter_db_optr {SCreateDBInfo} -alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} +%type topic_optr {SCreateDbInfo} + +topic_optr(Y) ::= db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } +topic_optr(Y) ::= topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } + +%type alter_db_optr {SCreateDbInfo} +alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y); Y.dbType = TSDB_DB_TYPE_DEFAULT;} alter_db_optr(Y) ::= alter_db_optr(Z) replica(X). { Y = Z; Y.replica = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) quorum(X). { Y = Z; Y.quorum = strtol(X.z, NULL, 10); } @@ -274,20 +291,25 @@ alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = s alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = strtol(X.z, NULL, 10); } +%type alter_topic_optr {SCreateDbInfo} + +alter_topic_optr(Y) ::= alter_db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; } +alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); } + %type typename {TAOS_FIELD} typename(A) ::= ids(X). { X.type = 0; - tSqlSetColumnType (&A, &X); + tSetColumnType (&A, &X); } //define binary type, e.g., binary(10), nchar(10) typename(A) ::= ids(X) LP signed(Y) RP. { if (Y <= 0) { X.type = 0; - tSqlSetColumnType(&A, &X); + tSetColumnType(&A, &X); } else { X.type = -Y; // negative value of name length - tSqlSetColumnType(&A, &X); + tSetColumnType(&A, &X); } } @@ -295,7 +317,7 @@ typename(A) ::= ids(X) LP signed(Y) RP. { typename(A) ::= ids(X) UNSIGNED(Z). { X.type = 0; X.n = ((Z.z + Z.n) - X.z); - tSqlSetColumnType (&A, &X); + tSetColumnType (&A, &X); } %type signed {int64_t} @@ -304,13 +326,15 @@ signed(A) ::= PLUS INTEGER(X). { A = strtol(X.z, NULL, 10); } signed(A) ::= MINUS INTEGER(X). { A = -strtol(X.z, NULL, 10);} ////////////////////////////////// The CREATE TABLE statement /////////////////////////////// -cmd ::= CREATE TABLE create_table_args. {} +cmd ::= CREATE TABLE create_table_args. {} +cmd ::= CREATE TABLE create_stable_args. {} +cmd ::= CREATE STABLE create_stable_args. {} cmd ::= CREATE TABLE create_table_list(Z). { pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = Z;} -%type create_table_list{SCreateTableSQL*} +%type create_table_list{SCreateTableSql*} %destructor create_table_list{destroyCreateTableSql($$);} create_table_list(A) ::= create_from_stable(Z). { - SCreateTableSQL* pCreateTable = calloc(1, sizeof(SCreateTableSQL)); + SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); taosArrayPush(pCreateTable->childTableInfo, &Z); @@ -323,9 +347,9 @@ create_table_list(A) ::= create_table_list(X) create_from_stable(Z). { A = X; } -%type create_table_args{SCreateTableSQL*} +%type create_table_args{SCreateTableSql*} create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { - A = tSetCreateSqlElems(X, NULL, NULL, TSQL_CREATE_TABLE); + A = tSetCreateTableInfo(X, NULL, NULL, TSQL_CREATE_TABLE); setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); V.n += Z.n; @@ -333,8 +357,9 @@ create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { } // create super table -create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { - A = tSetCreateSqlElems(X, Y, NULL, TSQL_CREATE_STABLE); +%type create_stable_args{SCreateTableSql*} +create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { + A = tSetCreateTableInfo(X, Y, NULL, TSQL_CREATE_STABLE); setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); V.n += Z.n; @@ -347,13 +372,24 @@ create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TA create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) TAGS LP tagitemlist(Y) RP. { X.n += F.n; V.n += Z.n; - A = createNewChildTableInfo(&X, Y, &V, &U); + A = createNewChildTableInfo(&X, NULL, Y, &V, &U); } +create_from_stable(A) ::= ifnotexists(U) ids(V) cpxName(Z) USING ids(X) cpxName(F) LP tagNamelist(P) RP TAGS LP tagitemlist(Y) RP. { + X.n += F.n; + V.n += Z.n; + A = createNewChildTableInfo(&X, P, Y, &V, &U); +} + +%type tagNamelist{SArray*} +%destructor tagNamelist {taosArrayDestroy($$);} +tagNamelist(A) ::= tagNamelist(X) COMMA ids(Y). {taosArrayPush(X, &Y); A = X; } +tagNamelist(A) ::= ids(X). {A = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(A, &X);} + // create stream // create table table_name as select count(*) from super_table_name interval(time) create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { - A = tSetCreateSqlElems(NULL, NULL, S, TSQL_CREATE_STREAM); + A = tSetCreateTableInfo(NULL, NULL, S, TSQL_CREATE_STREAM); setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); V.n += Z.n; @@ -369,7 +405,7 @@ columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(T // The information used for a column is the name and type of column: // tinyint smallint int bigint float double bool timestamp binary(x) nchar(x) column(A) ::= ids(X) typename(Y). { - tSqlSetColumnInfo(&A, &X, &Y); + tSetColumnInfo(&A, &X, &Y); } %type tagitemlist {SArray*} @@ -414,91 +450,91 @@ tagitem(A) ::= PLUS(X) FLOAT(Y). { } //////////////////////// The SELECT statement ///////////////////////////////// -%type select {SQuerySQL*} -%destructor select {doDestroyQuerySql($$);} -select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { - A = tSetQuerySqlElems(&T, W, X, Y, P, Z, &K, &S, F, &L, &G); +%type select {SQuerySqlNode*} +%destructor select {destroyQuerySqlNode($$);} +select(A) ::= SELECT(T) selcollist(W) from(X) where_opt(Y) interval_opt(K) session_option(H) fill_opt(F) sliding_opt(S) groupby_opt(P) orderby_opt(Z) having_opt(N) slimit_opt(G) limit_opt(L). { + A = tSetQuerySqlNode(&T, W, X, Y, P, Z, &K, &H, &S, F, &L, &G); } +select(A) ::= LP select(B) RP. {A = B;} + %type union {SSubclauseInfo*} %destructor union {destroyAllSelectClause($$);} - union(Y) ::= select(X). { Y = setSubclause(NULL, X); } -union(Y) ::= LP union(X) RP. { Y = X; } union(Y) ::= union(Z) UNION ALL select(X). { Y = appendSelectClause(Z, X); } -union(Y) ::= union(Z) UNION ALL LP select(X) RP. { Y = appendSelectClause(Z, X); } cmd ::= union(X). { setSqlInfo(pInfo, X, NULL, TSDB_SQL_SELECT); } // Support for the SQL exprssion without from & where subclauses, e.g., -// select current_database(), -// select server_version(), select client_version(), -// select server_state(); +// select current_database() +// select server_version() +// select client_version() +// select server_state() select(A) ::= SELECT(T) selcollist(W). { - A = tSetQuerySqlElems(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + A = tSetQuerySqlNode(&T, W, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } // selcollist is a list of expressions that are to become the return // values of the SELECT statement. The "*" in statements like // "SELECT * FROM ..." is encoded as a special expression with an opcode of TK_ALL. -%type selcollist {tSQLExprList*} +%type selcollist {SArray*} %destructor selcollist {tSqlExprListDestroy($$);} -%type sclp {tSQLExprList*} +%type sclp {SArray*} %destructor sclp {tSqlExprListDestroy($$);} sclp(A) ::= selcollist(X) COMMA. {A = X;} sclp(A) ::= . {A = 0;} -selcollist(A) ::= sclp(P) expr(X) as(Y). { - A = tSqlExprListAppend(P, X, Y.n?&Y:0); +selcollist(A) ::= sclp(P) distinct(Z) expr(X) as(Y). { + A = tSqlExprListAppend(P, X, Z.n? &Z:0, Y.n?&Y:0); } selcollist(A) ::= sclp(P) STAR. { - tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); - A = tSqlExprListAppend(P, pNode, 0); + tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); + A = tSqlExprListAppend(P, pNode, 0, 0); } // An option "AS " phrase that can follow one of the expressions that // define the result set, or one of the tables in the FROM clause. -// %type as {SStrToken} as(X) ::= AS ids(Y). { X = Y; } as(X) ::= ids(Y). { X = Y; } as(X) ::= . { X.n = 0; } +%type distinct {SStrToken} +distinct(X) ::= DISTINCT(Y). { X = Y; } +distinct(X) ::= . { X.n = 0;} + // A complete FROM clause. -%type from {SArray*} -// current not support query from no-table +%type from {SFromInfo*} from(A) ::= FROM tablelist(X). {A = X;} +from(A) ::= FROM LP union(Y) RP. {A = Y;} %type tablelist {SArray*} tablelist(A) ::= ids(X) cpxName(Y). { toTSDBType(X.type); X.n += Y.n; - A = tVariantListAppendToken(NULL, &X, -1); - A = tVariantListAppendToken(A, &X, -1); // table alias name + A = setTableNameList(NULL, &X, NULL); } tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { toTSDBType(X.type); toTSDBType(Z.type); X.n += Y.n; - A = tVariantListAppendToken(NULL, &X, -1); - A = tVariantListAppendToken(A, &Z, -1); + A = setTableNameList(NULL, &X, &Z); } tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { toTSDBType(X.type); X.n += Z.n; - A = tVariantListAppendToken(Y, &X, -1); - A = tVariantListAppendToken(A, &X, -1); + A = setTableNameList(Y, &X, NULL); } tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { toTSDBType(X.type); toTSDBType(F.type); X.n += Z.n; - A = tVariantListAppendToken(Y, &X, -1); - A = tVariantListAppendToken(A, &F, -1); + + A = setTableNameList(Y, &X, &F); } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" @@ -506,13 +542,21 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { tmvar(A) ::= VARIABLE(X). {A = X;} %type interval_opt {SIntervalVal} -interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.offset.z = NULL; N.offset.type = 0;} -interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(O) RP. {N.interval = E; N.offset = O;} +interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N.interval = E; N.offset.n = 0;} +interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(X) RP. {N.interval = E; N.offset = X;} interval_opt(N) ::= . {memset(&N, 0, sizeof(N));} +%type session_option {SSessionWindowVal} +session_option(X) ::= . {X.col.n = 0; X.gap.n = 0;} +session_option(X) ::= SESSION LP ids(V) cpxName(Z) COMMA tmvar(Y) RP. { + V.n += Z.n; + X.col = V; + X.gap = Y; +} + %type fill_opt {SArray*} %destructor fill_opt {taosArrayDestroy($$);} -fill_opt(N) ::= . {N = 0; } +fill_opt(N) ::= . { N = 0; } fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. { tVariant A = {0}; toTSDBType(Y.type); @@ -582,7 +626,7 @@ grouplist(A) ::= item(X). { } //having clause, ignore the input condition in having -%type having_opt {tSQLExpr*} +%type having_opt {tSqlExpr*} %destructor having_opt {tSqlExprDestroy($$);} having_opt(A) ::=. {A = 0;} having_opt(A) ::= HAVING expr(X). {A = X;} @@ -604,7 +648,7 @@ slimit_opt(A) ::= SLIMIT signed(X) SOFFSET signed(Y). slimit_opt(A) ::= SLIMIT signed(X) COMMA signed(Y). {A.limit = Y; A.offset = X;} -%type where_opt {tSQLExpr*} +%type where_opt {tSqlExpr*} %destructor where_opt {tSqlExprDestroy($$);} where_opt(A) ::= . {A = 0;} @@ -612,25 +656,26 @@ where_opt(A) ::= WHERE expr(X). {A = X;} /////////////////////////// Expression Processing ///////////////////////////// // -%type expr {tSQLExpr*} +%type expr {tSqlExpr*} %destructor expr {tSqlExprDestroy($$);} expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->token.z = X.z; A->token.n = (Z.z - X.z + 1);} -expr(A) ::= ID(X). { A = tSqlExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprIdValueCreate(&X, TK_ID);} -expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprIdValueCreate(&X, TK_ALL);} - -expr(A) ::= INTEGER(X). { A = tSqlExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprIdValueCreate(&X, TK_INTEGER);} -expr(A) ::= FLOAT(X). { A = tSqlExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprIdValueCreate(&X, TK_FLOAT);} -expr(A) ::= STRING(X). { A = tSqlExprIdValueCreate(&X, TK_STRING);} -expr(A) ::= NOW(X). { A = tSqlExprIdValueCreate(&X, TK_NOW); } -expr(A) ::= VARIABLE(X). { A = tSqlExprIdValueCreate(&X, TK_VARIABLE);} -expr(A) ::= BOOL(X). { A = tSqlExprIdValueCreate(&X, TK_BOOL);} +expr(A) ::= ID(X). { A = tSqlExprCreateIdValue(&X, TK_ID);} +expr(A) ::= ID(X) DOT ID(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ID);} +expr(A) ::= ID(X) DOT STAR(Y). { X.n += (1+Y.n); A = tSqlExprCreateIdValue(&X, TK_ALL);} + +expr(A) ::= INTEGER(X). { A = tSqlExprCreateIdValue(&X, TK_INTEGER);} +expr(A) ::= MINUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} +expr(A) ::= PLUS(X) INTEGER(Y). { X.n += Y.n; X.type = TK_INTEGER; A = tSqlExprCreateIdValue(&X, TK_INTEGER);} +expr(A) ::= FLOAT(X). { A = tSqlExprCreateIdValue(&X, TK_FLOAT);} +expr(A) ::= MINUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} +expr(A) ::= PLUS(X) FLOAT(Y). { X.n += Y.n; X.type = TK_FLOAT; A = tSqlExprCreateIdValue(&X, TK_FLOAT);} +expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(&X, TK_STRING);} +expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(&X, TK_NOW); } +expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} +expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);} +expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);} // ordinary functions: min(x), max(x), top(k, 20) expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSqlExprCreateFunction(Y, &X, &E, X.type); } @@ -650,6 +695,8 @@ expr(A) ::= expr(X) GE expr(Y). {A = tSqlExprCreate(X, Y, TK_GE);} expr(A) ::= expr(X) NE expr(Y). {A = tSqlExprCreate(X, Y, TK_NE);} expr(A) ::= expr(X) EQ expr(Y). {A = tSqlExprCreate(X, Y, TK_EQ);} +expr(A) ::= expr(X) BETWEEN expr(Y) AND expr(Z). { tSqlExpr* X2 = tSqlExprClone(X); A = tSqlExprCreate(tSqlExprCreate(X, Y, TK_GE), tSqlExprCreate(X2, Z, TK_LE), TK_AND);} + expr(A) ::= expr(X) AND expr(Y). {A = tSqlExprCreate(X, Y, TK_AND);} expr(A) ::= expr(X) OR expr(Y). {A = tSqlExprCreate(X, Y, TK_OR); } @@ -664,26 +711,29 @@ expr(A) ::= expr(X) REM expr(Y). {A = tSqlExprCreate(X, Y, TK_REM); } expr(A) ::= expr(X) LIKE expr(Y). {A = tSqlExprCreate(X, Y, TK_LIKE); } //in expression -expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSQLExpr*)Y, TK_IN); } +expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSqlExprCreate(X, (tSqlExpr*)Y, TK_IN); } -%type exprlist {tSQLExprList*} +%type exprlist {SArray*} %destructor exprlist {tSqlExprListDestroy($$);} -%type expritem {tSQLExpr*} +%type expritem {tSqlExpr*} %destructor expritem {tSqlExprDestroy($$);} -exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0);} -exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0);} +exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSqlExprListAppend(X,Y,0, 0);} +exprlist(A) ::= expritem(X). {A = tSqlExprListAppend(0,X,0, 0);} expritem(A) ::= expr(X). {A = X;} expritem(A) ::= . {A = 0;} ///////////////////////////////////reset query cache////////////////////////////////////// -cmd ::= RESET QUERY CACHE. { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} +cmd ::= RESET QUERY CACHE. { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} + +///////////////////////////////////sync replica database////////////////////////////////// +cmd ::= SYNCDB ids(X) REPLICA.{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &X);} ///////////////////////////////////ALTER TABLE statement////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { X.n += F.n; - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -693,14 +743,14 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A). { toTSDBType(A.type); SArray* K = tVariantListAppendToken(NULL, &A, -1); - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } //////////////////////////////////ALTER TAGS statement///////////////////////////////////// cmd ::= ALTER TABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { X.n += Y.n; - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { @@ -709,7 +759,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y). { toTSDBType(Y.type); SArray* A = tVariantListAppendToken(NULL, &Y, -1); - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -722,7 +772,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { toTSDBType(Z.type); A = tVariantListAppendToken(A, &Z, -1); - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -733,7 +783,54 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z). { SArray* A = tVariantListAppendToken(NULL, &Y, -1); A = tVariantListAppend(A, &Z, -1); - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + + +///////////////////////////////////ALTER STABLE statement////////////////////////////////// +cmd ::= ALTER STABLE ids(X) cpxName(F) ADD COLUMN columnlist(A). { + X.n += F.n; + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + +cmd ::= ALTER STABLE ids(X) cpxName(F) DROP COLUMN ids(A). { + X.n += F.n; + + toTSDBType(A.type); + SArray* K = tVariantListAppendToken(NULL, &A, -1); + + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + +//////////////////////////////////ALTER TAGS statement///////////////////////////////////// +cmd ::= ALTER STABLE ids(X) cpxName(Y) ADD TAG columnlist(A). { + X.n += Y.n; + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, A, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} +cmd ::= ALTER STABLE ids(X) cpxName(Z) DROP TAG ids(Y). { + X.n += Z.n; + + toTSDBType(Y.type); + SArray* A = tVariantListAppendToken(NULL, &Y, -1); + + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + +cmd ::= ALTER STABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). { + X.n += F.n; + + toTSDBType(Y.type); + SArray* A = tVariantListAppendToken(NULL, &Y, -1); + + toTSDBType(Z.type); + A = tVariantListAppendToken(A, &Z, -1); + + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&X, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } @@ -745,6 +842,4 @@ cmd ::= KILL QUERY INTEGER(X) COLON(Z) INTEGER(Y). {X.n += (Z.n + Y.n); s %fallback ID ABORT AFTER ASC ATTACH BEFORE BEGIN CASCADE CLUSTER CONFLICT COPY DATABASE DEFERRED DELIMITERS DESC DETACH EACH END EXPLAIN FAIL FOR GLOB IGNORE IMMEDIATE INITIALLY INSTEAD LIKE MATCH KEY OF OFFSET RAISE REPLACE RESTRICT ROW STATEMENT TRIGGER VIEW ALL - COUNT SUM AVG MIN MAX FIRST LAST TOP BOTTOM STDDEV PERCENTILE APERCENTILE LEASTSQUARES HISTOGRAM DIFF - SPREAD TWA INTERP LAST_ROW RATE IRATE SUM_RATE SUM_IRATE AVG_RATE AVG_IRATE TBID NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT - METRIC TBNAME JOIN METRICS STABLE NULL INSERT INTO VALUES. + NOW IPTOKEN SEMI NONE PREV LINEAR IMPORT TBNAME JOIN STABLE NULL INSERT INTO VALUES. \ No newline at end of file diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 3c7fd794bf88e0fee53e7d7c40bdf91a832b89f3..f18d093b89d48419e9707d606a289b3168fea8bf 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -14,22 +14,23 @@ */ #include "os.h" -#include "qAst.h" +#include "taosdef.h" +#include "taosmsg.h" +#include "texpr.h" +#include "ttype.h" +#include "tsdb.h" + +#include "qAggMain.h" #include "qFill.h" #include "qHistogram.h" #include "qPercentile.h" #include "qTsbuf.h" -#include "taosdef.h" -#include "taosmsg.h" #include "queryLog.h" -#include "tscSubquery.h" -#include "tsqlfunction.h" -#include "ttype.h" -#define GET_INPUT_DATA_LIST(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) +#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) #define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) -#define GET_TS_LIST(x) ((TSKEY*)&((x)->ptsList[(x)->startOffset])) +#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) #define GET_TRUE_DATA_TYPE() \ @@ -90,8 +91,9 @@ typedef struct SSpreadInfo { typedef struct SSumInfo { union { - int64_t isum; - double dsum; + int64_t isum; + uint64_t usum; + double dsum; }; int8_t hasResult; } SSumInfo; @@ -109,6 +111,11 @@ typedef struct SStddevInfo { int8_t stage; } SStddevInfo; +typedef struct SStddevdstInfo { + int64_t num; + double res; +} SStddevdstInfo; + typedef struct SFirstLastInfo { int8_t hasResult; TSKEY ts; @@ -183,6 +190,11 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); *interBytes = 0; return TSDB_CODE_SUCCESS; + } else if (functionId == TSDB_FUNC_BLKINFO) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = 16384; + *interBytes = 0; + return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_COUNT) { @@ -201,7 +213,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI if (functionId == TSDB_FUNC_TS_COMP) { *type = TSDB_DATA_TYPE_BINARY; - *bytes = sizeof(int32_t); // this results is compressed ts data + *bytes = 1; // this results is compressed ts data, only one byte *interBytes = POINTER_BYTES; return TSDB_CODE_SUCCESS; } @@ -335,6 +347,11 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; *interBytes = dataBytes; + } else if (functionId == TSDB_FUNC_STDDEV_DST) { + *type = TSDB_DATA_TYPE_BINARY; + *bytes = sizeof(SStddevdstInfo); + *interBytes = (*bytes); + } else { return TSDB_CODE_TSC_INVALID_SQL; } @@ -342,6 +359,22 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } +// TODO use hash table +int32_t isValidFunction(const char* name, int32_t len) { + for(int32_t i = 0; i <= TSDB_FUNC_BLKINFO; ++i) { + int32_t nameLen = (int32_t) strlen(aAggs[i].name); + if (len != nameLen) { + continue; + } + + if (strncasecmp(aAggs[i].name, name, len) == 0) { + return i; + } + } + + return -1; +} + // set the query flag to denote that query is completed static void no_next_step(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -354,7 +387,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { return false; } - memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); + memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes); initResultInfo(pResInfo, pCtx->interBufBytes); return true; } @@ -369,11 +402,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { static void function_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->aOutputBuf, pCtx->outputType); - } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - } + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } doFinalizer(pCtx); @@ -404,10 +433,7 @@ static void count_function(SQLFunctionCtx *pCtx) { numOfElem += 1; } } else { - /* - * when counting on the primary time stamp column and no statistics data is provided, - * simple use the size value - */ + //when counting on the primary time stamp column and no statistics data is presented, use the size value directly. numOfElem = pCtx->size; } } @@ -416,7 +442,7 @@ static void count_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } - *((int64_t *)pCtx->aOutputBuf) += numOfElem; + *((int64_t *)pCtx->pOutput) += numOfElem; SET_VAL(pCtx, numOfElem, 1); } @@ -427,7 +453,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { } SET_VAL(pCtx, 1, 1); - *((int64_t *)pCtx->aOutputBuf) += pCtx->size; + *((int64_t *)pCtx->pOutput) += pCtx->size; // do not need it actually SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -437,7 +463,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void count_func_merge(SQLFunctionCtx *pCtx) { int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { - *((int64_t *)pCtx->aOutputBuf) += pData[i]; + *((int64_t *)pCtx->pOutput) += pData[i]; } SET_VAL(pCtx, pCtx->size, 1); @@ -452,7 +478,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { * @param filterCols * @return */ -int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return BLK_DATA_NO_NEEDED; } else { @@ -460,7 +486,7 @@ int32_t count_load_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32 } } -int32_t no_data_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_NO_NEEDED; } @@ -519,13 +545,13 @@ static void do_sum(SQLFunctionCtx *pCtx) { assert(pCtx->size >= pCtx->preAggVals.statis.numOfNull); if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t *retVal = (int64_t *)pCtx->aOutputBuf; + int64_t *retVal = (int64_t *)pCtx->pOutput; *retVal += pCtx->preAggVals.statis.sum; } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t *retVal = (uint64_t *)pCtx->aOutputBuf; + uint64_t *retVal = (uint64_t *)pCtx->pOutput; *retVal += (uint64_t)pCtx->preAggVals.statis.sum; - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double*) pCtx->aOutputBuf; + } else if (IS_FLOAT_TYPE(pCtx->inputType)) { + double *retVal = (double*) pCtx->pOutput; *retVal += GET_DOUBLE_VAL((const char*)&(pCtx->preAggVals.statis.sum)); } } else { // computing based on the true data block @@ -533,7 +559,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { notNullElems = 0; if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { - int64_t *retVal = (int64_t *)pCtx->aOutputBuf; + int64_t *retVal = (int64_t *)pCtx->pOutput; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); @@ -545,22 +571,22 @@ static void do_sum(SQLFunctionCtx *pCtx) { LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); } } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { - uint64_t *retVal = (uint64_t *)pCtx->aOutputBuf; + uint64_t *retVal = (uint64_t *)pCtx->pOutput; - if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { + if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { LIST_ADD_N(*retVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { LIST_ADD_N(*retVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { LIST_ADD_N(*retVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { LIST_ADD_N(*retVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType); } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = (double *)pCtx->aOutputBuf; + double *retVal = (double *)pCtx->pOutput; LIST_ADD_N(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double *)pCtx->aOutputBuf; + double *retVal = (double *)pCtx->pOutput; LIST_ADD_N(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } @@ -580,7 +606,7 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { } SET_VAL(pCtx, 1, 1); - int64_t *res = (int64_t*) pCtx->aOutputBuf; + int64_t *res = (int64_t*) pCtx->pOutput; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { *res += GET_INT8_VAL(pData); @@ -590,11 +616,23 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { *res += GET_INT32_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { *res += GET_INT64_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + uint64_t *r = (uint64_t *)pCtx->pOutput; + *r += GET_UINT8_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + uint64_t *r = (uint64_t *)pCtx->pOutput; + *r += GET_UINT16_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + uint64_t *r = (uint64_t *)pCtx->pOutput; + *r += GET_UINT32_VAL(pData); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + uint64_t *r = (uint64_t *)pCtx->pOutput; + *r += GET_UINT64_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = (double*) pCtx->aOutputBuf; + double *retVal = (double*) pCtx->pOutput; *retVal += GET_DOUBLE_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = (double*) pCtx->aOutputBuf; + double *retVal = (double*) pCtx->pOutput; *retVal += GET_FLOAT_VAL(pData); } @@ -608,7 +646,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { // set the flag for super table query - SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; + SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; pSum->hasResult = DATA_SET_FLAG; } } @@ -619,7 +657,7 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { // keep the result data in output buffer, not in the intermediate buffer SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; + SSumInfo *pSum = (SSumInfo *)pCtx->pOutput; pSum->hasResult = DATA_SET_FLAG; } } @@ -639,18 +677,12 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { notNullElems++; - switch (type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: { - *(int64_t *)pCtx->aOutputBuf += pInput->isum; - break; - }; - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: { - *(double *)pCtx->aOutputBuf += pInput->dsum; - } + if (IS_SIGNED_NUMERIC_TYPE(type)) { + *(int64_t *)pCtx->pOutput += pInput->isum; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + *(uint64_t *) pCtx->pOutput += pInput->usum; + } else { + *(double *)pCtx->pOutput += pInput->dsum; } } @@ -662,16 +694,16 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { } } -static int32_t statisRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_STATIS_NEEDED; } -static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_ALL_NEEDED; } -// todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +// todo: if column in current data block are null, opt for this case +static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -684,7 +716,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, i } } -static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i64) { return BLK_DATA_NO_NEEDED; } @@ -696,43 +728,43 @@ static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, in } } -static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } // not initialized yet, it is the first block, load it. - if (pCtx->aOutputBuf == NULL) { + if (pCtx->pOutput == NULL) { return BLK_DATA_ALL_NEEDED; } - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->aOutputBuf is + // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { // data in current block is not earlier than current result - return (pInfo->ts <= start) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + return (pInfo->ts <= w->skey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; } } -static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) { +static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i64) { return BLK_DATA_NO_NEEDED; } // not initialized yet, it is the first block, load it. - if (pCtx->aOutputBuf == NULL) { + if (pCtx->pOutput == NULL) { return BLK_DATA_ALL_NEEDED; } - // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->aOutputBuf is + // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid - SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { - return (pInfo->ts > end) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; + return (pInfo->ts > w->ekey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED; } } @@ -801,7 +833,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } @@ -848,14 +880,14 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_func_merge(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - double *sum = (double*) pCtx->aOutputBuf; + double *sum = (double*) pCtx->pOutput; char *input = GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -881,21 +913,21 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } - *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo); + *(double *)pCtx->pOutput = (*(double *)pCtx->pOutput) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo); } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY assert(IS_NUMERIC_TYPE(pCtx->inputType)); SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pAvgInfo->num == 0) { // all data are NULL or empty table - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } - *(double *)pCtx->aOutputBuf = pAvgInfo->sum / pAvgInfo->num; + *(double *)pCtx->pOutput = pAvgInfo->sum / pAvgInfo->num; } // cannot set the numOfIteratedElems again since it is set during previous iteration @@ -934,9 +966,9 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, * * The following codes of 3 lines will be removed later. */ - if (index < 0 || index >= pCtx->size + pCtx->startOffset) { - index = 0; - } +// if (index < 0 || index >= pCtx->size + pCtx->startOffset) { +// index = 0; +// } // the index is the original position, not the relative position key = pCtx->ptsList[index]; @@ -1046,7 +1078,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, TYPED_LOOPCHECK_N(uint16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { TYPED_LOOPCHECK_N(uint32_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); - } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { TYPED_LOOPCHECK_N(uint64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { @@ -1065,34 +1097,34 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) { switch (type) { case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->aOutputBuf) = INT8_MAX; + *((int8_t *)pCtx->pOutput) = INT8_MAX; break; case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *) pCtx->aOutputBuf = UINT8_MAX; + *(uint8_t *) pCtx->pOutput = UINT8_MAX; break; case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->aOutputBuf) = INT16_MAX; + *((int16_t *)pCtx->pOutput) = INT16_MAX; break; case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->aOutputBuf) = UINT16_MAX; + *((uint16_t *)pCtx->pOutput) = UINT16_MAX; break; case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->aOutputBuf) = INT32_MAX; + *((int32_t *)pCtx->pOutput) = INT32_MAX; break; case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->aOutputBuf) = UINT32_MAX; + *((uint32_t *)pCtx->pOutput) = UINT32_MAX; break; case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->aOutputBuf) = INT64_MAX; + *((int64_t *)pCtx->pOutput) = INT64_MAX; break; case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->aOutputBuf) = UINT64_MAX; + *((uint64_t *)pCtx->pOutput) = UINT64_MAX; break; case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->aOutputBuf) = FLT_MAX; + *((float *)pCtx->pOutput) = FLT_MAX; break; case TSDB_DATA_TYPE_DOUBLE: - *((double *)pCtx->aOutputBuf) = DBL_MAX; + *((double *)pCtx->pOutput) = DBL_MAX; break; default: qError("illegal data type:%d in min/max query", pCtx->inputType); @@ -1110,34 +1142,34 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { switch (type) { case TSDB_DATA_TYPE_INT: - *((int32_t *)pCtx->aOutputBuf) = INT32_MIN; + *((int32_t *)pCtx->pOutput) = INT32_MIN; break; case TSDB_DATA_TYPE_UINT: - *((uint32_t *)pCtx->aOutputBuf) = 0; + *((uint32_t *)pCtx->pOutput) = 0; break; case TSDB_DATA_TYPE_FLOAT: - *((float *)pCtx->aOutputBuf) = -FLT_MAX; + *((float *)pCtx->pOutput) = -FLT_MAX; break; case TSDB_DATA_TYPE_DOUBLE: - *((double *)pCtx->aOutputBuf) = -DBL_MAX; + *((double *)pCtx->pOutput) = -DBL_MAX; break; case TSDB_DATA_TYPE_BIGINT: - *((int64_t *)pCtx->aOutputBuf) = INT64_MIN; + *((int64_t *)pCtx->pOutput) = INT64_MIN; break; case TSDB_DATA_TYPE_UBIGINT: - *((uint64_t *)pCtx->aOutputBuf) = 0; + *((uint64_t *)pCtx->pOutput) = 0; break; case TSDB_DATA_TYPE_SMALLINT: - *((int16_t *)pCtx->aOutputBuf) = INT16_MIN; + *((int16_t *)pCtx->pOutput) = INT16_MIN; break; case TSDB_DATA_TYPE_USMALLINT: - *((uint16_t *)pCtx->aOutputBuf) = 0; + *((uint16_t *)pCtx->pOutput) = 0; break; case TSDB_DATA_TYPE_TINYINT: - *((int8_t *)pCtx->aOutputBuf) = INT8_MIN; + *((int8_t *)pCtx->pOutput) = INT8_MIN; break; case TSDB_DATA_TYPE_UTINYINT: - *((uint8_t *)pCtx->aOutputBuf) = 0; + *((uint8_t *)pCtx->pOutput) = 0; break; default: qError("illegal data type:%d in min/max query", pCtx->inputType); @@ -1151,7 +1183,7 @@ static bool max_func_setup(SQLFunctionCtx *pCtx) { */ static void min_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->aOutputBuf, 1, ¬NullElems); + minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); SET_VAL(pCtx, notNullElems, 1); @@ -1161,14 +1193,14 @@ static void min_function(SQLFunctionCtx *pCtx) { // set the flag for super table query if (pCtx->stableQuery) { - *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; + *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; } } } static void max_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - minMax_function(pCtx, pCtx->aOutputBuf, 0, ¬NullElems); + minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); SET_VAL(pCtx, notNullElems, 1); @@ -1178,7 +1210,7 @@ static void max_function(SQLFunctionCtx *pCtx) { // set the flag for super table query if (pCtx->stableQuery) { - *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; + *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG; } } } @@ -1244,7 +1276,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp } static void min_func_merge(SQLFunctionCtx *pCtx) { - int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 1); + int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1255,7 +1287,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { } static void max_func_merge(SQLFunctionCtx *pCtx) { - int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->aOutputBuf, 0); + int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1271,32 +1303,32 @@ static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin int32_t num = 0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { - int8_t *output = (int8_t *)pCtx->aOutputBuf; + int8_t *output = (int8_t *)pCtx->pOutput; int8_t i = GET_INT8_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *output = (int16_t*) pCtx->aOutputBuf; + int16_t *output = (int16_t*) pCtx->pOutput; int16_t i = GET_INT16_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *output = (int32_t*) pCtx->aOutputBuf; + int32_t *output = (int32_t*) pCtx->pOutput; int32_t i = GET_INT32_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *output = (int64_t*) pCtx->aOutputBuf; + int64_t *output = (int64_t*) pCtx->pOutput; int64_t i = GET_INT64_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *output = (float*) pCtx->aOutputBuf; + float *output = (float*) pCtx->pOutput; float i = GET_FLOAT_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *output = (double*) pCtx->aOutputBuf; + double *output = (double*) pCtx->pOutput; double i = GET_DOUBLE_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); @@ -1316,7 +1348,7 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; + char *flag = pCtx->pOutput + pCtx->inputBytes; *flag = DATA_SET_FLAG; } } @@ -1332,21 +1364,36 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { - char *flag = pCtx->aOutputBuf + pCtx->inputBytes; + char *flag = pCtx->pOutput + pCtx->inputBytes; *flag = DATA_SET_FLAG; } } -#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, tsdbType) \ - for (int32_t i = 0; i < (ctx)->size; ++i) { \ - if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], tsdbType)) { \ - continue; \ - } \ - (r) += POW2(((type *)d)[i] - (delta)); \ +#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num) \ + for (int32_t i = 0; i < (ctx)->size; ++i) { \ + if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \ + continue; \ + } \ + (num) += 1; \ + (r) += POW2(((type *)d)[i] - (delta)); \ } static void stddev_function(SQLFunctionCtx *pCtx) { - SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); + + if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { + pStd->stage++; + avg_finalizer(pCtx); + + pResInfo->initialized = true; // set it initialized to avoid re-initialization + + // save average value into tmpBuf, for second stage scan + SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); + + pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); + assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); + } if (pStd->stage == 0) { // the first stage is to calculate average value @@ -1358,35 +1405,53 @@ static void stddev_function(SQLFunctionCtx *pCtx) { double avg = pStd->avg; void *pData = GET_INPUT_DATA_LIST(pCtx); - + int32_t num = 0; + switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pCtx->size; ++i) { if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { continue; } + num += 1; *retVal += POW2(((int32_t *)pData)[i] - avg); } break; } case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType); + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + LOOP_STDDEV_IMPL(uint8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UINT: { + LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); break; } default: @@ -1401,7 +1466,20 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { // the second stage to calculate standard deviation SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); - + + if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) { + pStd->stage++; + avg_finalizer(pCtx); + + pResInfo->initialized = true; // set it initialized to avoid re-initialization + + // save average value into tmpBuf, for second stage scan + SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); + + pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); + assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); + } + /* the first stage is to calculate average value */ if (pStd->stage == 0) { avg_function_f(pCtx, index); @@ -1438,6 +1516,22 @@ static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { pStd->res += POW2(GET_INT8_VAL(pData) - avg); break; } + case TSDB_DATA_TYPE_UINT: { + pStd->res += POW2(GET_UINT32_VAL(pData) - avg); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + pStd->res += POW2(GET_UINT64_VAL(pData) - avg); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + pStd->res += POW2(GET_UINT16_VAL(pData) - avg); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + pStd->res += POW2(GET_UINT8_VAL(pData) - avg); + break; + } default: qError("stddev function not support data type:%d", pCtx->inputType); } @@ -1469,7 +1563,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { // save average value into tmpBuf, for second stage scan SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); - pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf); + pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput); assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); } else { pResInfo->complete = true; @@ -1480,9 +1574,9 @@ static void stddev_finalizer(SQLFunctionCtx *pCtx) { SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } else { - double *retValue = (double *)pCtx->aOutputBuf; + double *retValue = (double *)pCtx->pOutput; *retValue = sqrt(pStd->res / pStd->num); SET_VAL(pCtx, 1, 1); } @@ -1490,6 +1584,230 @@ static void stddev_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } +////////////////////////////////////////////////////////////////////////////////////// +int32_t tsCompare(const void* p1, const void* p2) { + TSKEY k = *(TSKEY*)p1; + SResPair* pair = (SResPair*)p2; + + if (k == pair->key) { + return 0; + } else { + return k < pair->key? -1:1; + } +} + +static void stddev_dst_function(SQLFunctionCtx *pCtx) { + SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + // the second stage to calculate standard deviation + double *retVal = &pStd->res; + + // all data are null, no need to proceed + SArray* resList = (SArray*) pCtx->param[0].pz; + if (resList == NULL) { + return; + } + + // find the correct group average results according to the tag value + int32_t len = (int32_t) taosArrayGetSize(resList); + assert(len > 0); + + double avg = 0; + if (len == 1) { + SResPair* p = taosArrayGet(resList, 0); + avg = p->avg; + } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result + SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); + if (p == NULL) { + return; + } + + avg = p->avg; + } + + void *pData = GET_INPUT_DATA_LIST(pCtx); + int32_t num = 0; + + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_INT: { + for (int32_t i = 0; i < pCtx->size; ++i) { + if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { + continue; + } + num += 1; + *retVal += POW2(((int32_t *)pData)[i] - avg); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_TINYINT: { + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UINT: { + LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + default: + qError("stddev function not support data type:%d", pCtx->inputType); + } + + pStd->num += num; + SET_VAL(pCtx, num, 1); + + // copy to the final output buffer for super table + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); +} + +static void stddev_dst_function_f(SQLFunctionCtx *pCtx, int32_t index) { + void *pData = GET_INPUT_DATA(pCtx, index); + if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { + return; + } + + // the second stage to calculate standard deviation + SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + double *retVal = &pStd->res; + + // all data are null, no need to proceed + SArray* resList = (SArray*) pCtx->param[0].pz; + if (resList == NULL) { + return; + } + + // find the correct group average results according to the tag value + int32_t len = (int32_t) taosArrayGetSize(resList); + assert(len > 0); + + double avg = 0; + if (len == 1) { + SResPair* p = taosArrayGet(resList, 0); + avg = p->avg; + } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result + SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); + assert(p != NULL); + + avg = p->avg; + } + + int32_t num = 0; + switch (pCtx->inputType) { + case TSDB_DATA_TYPE_INT: { + for (int32_t i = 0; i < pCtx->size; ++i) { + if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { + continue; + } + num += 1; + *retVal += POW2(((int32_t *)pData)[i] - avg); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_TINYINT: { + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UINT: { + LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); + break; + } + default: + qError("stddev function not support data type:%d", pCtx->inputType); + } + + pStd->num += num; + SET_VAL(pCtx, num, 1); + + // copy to the final output buffer for super table + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); +} + + +static void stddev_dst_merge(SQLFunctionCtx *pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); + + char *input = GET_INPUT_DATA_LIST(pCtx); + + for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { + SStddevdstInfo *pInput = (SStddevdstInfo *)input; + if (pInput->num == 0) { // current input is null + continue; + } + + pRes->num += pInput->num; + pRes->res += pInput->res; + } +} + +static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { + SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + if (pStd->num <= 0) { + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + } else { + double *retValue = (double *)pCtx->pOutput; + *retValue = sqrt(pStd->res / pStd->num); + SET_VAL(pCtx, 1, 1); + } + + doFinalizer(pCtx); +} + ////////////////////////////////////////////////////////////////////////////////////// static bool first_last_function_setup(SQLFunctionCtx *pCtx) { if (!function_setup(pCtx)) { @@ -1505,7 +1823,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx) { // todo opt for null block static void first_function(SQLFunctionCtx *pCtx) { - if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) { + if (pCtx->order == TSDB_ORDER_DESC /*|| pCtx->preAggVals.dataBlockLoaded == false*/) { return; } @@ -1518,7 +1836,7 @@ static void first_function(SQLFunctionCtx *pCtx) { continue; } - memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); + memcpy(pCtx->pOutput, data, pCtx->inputBytes); TSKEY k = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, k); @@ -1545,7 +1863,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { } SET_VAL(pCtx, 1, 1); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); @@ -1558,10 +1876,10 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; @@ -1579,7 +1897,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { * 1. data block that are not loaded * 2. scan data files in desc order */ - if (pCtx->order == TSDB_ORDER_DESC || pCtx->preAggVals.dataBlockLoaded == false) { + if (pCtx->order == TSDB_ORDER_DESC/* || pCtx->preAggVals.dataBlockLoaded == false*/) { return; } @@ -1630,7 +1948,7 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { // The param[1] is used to keep the initial value of max ts value if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 > pInput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); + memcpy(pCtx->pOutput, pData, pCtx->outputBytes); pCtx->param[1].i64 = pInput->ts; pCtx->param[1].nType = pCtx->outputType; @@ -1650,7 +1968,7 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SQLFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i64 || pCtx->preAggVals.dataBlockLoaded == false) { + if (pCtx->order != pCtx->param[0].i64/* || pCtx->preAggVals.dataBlockLoaded == false*/) { return; } @@ -1663,7 +1981,8 @@ static void last_function(SQLFunctionCtx *pCtx) { continue; } } - memcpy(pCtx->aOutputBuf, data, pCtx->inputBytes); + + memcpy(pCtx->pOutput, data, pCtx->inputBytes); TSKEY ts = GET_TS_DATA(pCtx, i); DO_UPDATE_TAG_COLUMNS(pCtx, ts); @@ -1692,7 +2011,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->order == TSDB_ORDER_DESC) { SET_VAL(pCtx, 1, 1); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); TSKEY ts = GET_TS_DATA(pCtx, index); DO_UPDATE_TAG_COLUMNS(pCtx, ts); @@ -1707,7 +2026,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { char* buf = GET_ROWCELL_INTERBUF(pResInfo); if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); *(TSKEY*)buf = ts; DO_UPDATE_TAG_COLUMNS(pCtx, ts); @@ -1718,14 +2037,14 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); - SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) { #if defined(_DEBUG_VIEW) qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData); #endif - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); pInfo->hasResult = DATA_SET_FLAG; pInfo->ts = timestamp[index]; @@ -1742,13 +2061,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { return; } - // data block is discard, not loaded, do not need to check it - if (!pCtx->preAggVals.dataBlockLoaded) { - return; - } - int32_t notNullElems = 0; - for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *data = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(data, pCtx->inputType)) { @@ -1810,7 +2123,7 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { * the true last result */ if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 < pInput->ts) { - memcpy(pCtx->aOutputBuf, pData, pCtx->outputBytes); + memcpy(pCtx->pOutput, pData, pCtx->outputBytes); pCtx->param[1].i64 = pInput->ts; pCtx->param[1].nType = pCtx->outputType; @@ -1830,14 +2143,14 @@ static void last_row_function(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_DATA_LIST(pCtx); // assign the last element in current data block - assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); + assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the result to final result buffer in case of super table query if (pCtx->stableQuery) { - SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); + SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->pOutput + pCtx->inputBytes); pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1); pInfo1->hasResult = DATA_SET_FLAG; @@ -1854,12 +2167,7 @@ static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->aOutputBuf, pCtx->outputType); - } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); - } - + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } @@ -1901,26 +2209,29 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tVariant val = {0}; - tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); + tVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); tValuePair **pList = pInfo->res; assert(pList != NULL); if (pInfo->num < maxLen) { if (pInfo->num == 0 || - ((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && - val.i64 >= pList[pInfo->num - 1]->v.i64) || - ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && - val.dKey >= pList[pInfo->num - 1]->v.dKey)) { + (IS_SIGNED_NUMERIC_TYPE(type) && val.i64 >= pList[pInfo->num - 1]->v.i64) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 >= pList[pInfo->num - 1]->v.u64) || + (IS_FLOAT_TYPE(type) && val.dKey >= pList[pInfo->num - 1]->v.dKey)) { valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; - - if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { while (i >= 0 && pList[i]->v.i64 > val.i64) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); i -= 1; } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + while (i >= 0 && pList[i]->v.u64 > val.u64) { + VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); + i -= 1; + } } else { while (i >= 0 && pList[i]->v.dKey > val.dKey) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); @@ -1934,23 +2245,29 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, pInfo->num++; } else { int32_t i = 0; - - if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64 > pList[0]->v.i64) || - ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey > pList[0]->v.dKey)) { + + if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 > pList[0]->v.i64) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) || + (IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) { // find the appropriate the slot position - if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { while (i + 1 < maxLen && pList[i + 1]->v.i64 < val.i64) { VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); i += 1; } + } if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + while (i + 1 < maxLen && pList[i + 1]->v.u64 < val.u64) { + VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); + i += 1; + } } else { while (i + 1 < maxLen && pList[i + 1]->v.dKey < val.dKey) { VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); i += 1; } } - - valuePairAssign(pList[i], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage); + + valuePairAssign(pList[i], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); } } } @@ -1958,7 +2275,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tVariant val = {0}; - tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); + tVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); tValuePair **pList = pInfo->res; assert(pList != NULL); @@ -1969,11 +2286,16 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa } else { int32_t i = pInfo->num - 1; - if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { while (i >= 0 && pList[i]->v.i64 < val.i64) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); i -= 1; } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + while (i >= 0 && pList[i]->v.u64 < val.u64) { + VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); + i -= 1; + } } else { while (i >= 0 && pList[i]->v.dKey < val.dKey) { VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); @@ -1988,14 +2310,20 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa } else { int32_t i = 0; - if (((type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) && val.i64 < pList[0]->v.i64) || - ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey < pList[0]->v.dKey)) { + if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 < pList[0]->v.i64) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) || + (IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) { // find the appropriate the slot position - if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_BIGINT) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { while (i + 1 < maxLen && pList[i + 1]->v.i64 > val.i64) { VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); i += 1; } + } if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + while (i + 1 < maxLen && pList[i + 1]->v.u64 > val.u64) { + VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); + i += 1; + } } else { while (i + 1 < maxLen && pList[i + 1]->v.dKey > val.dKey) { VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); @@ -2025,19 +2353,24 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { tValuePair *pLeftElem = *(tValuePair **)pLeft; tValuePair *pRightElem = *(tValuePair **)pRight; - int32_t type = pLeftElem->v.nType; - if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + if (IS_FLOAT_TYPE(pLeftElem->v.nType)) { if (pLeftElem->v.dKey == pRightElem->v.dKey) { return 0; } else { return pLeftElem->v.dKey > pRightElem->v.dKey ? 1 : -1; } - } else { + } else if (IS_SIGNED_NUMERIC_TYPE(pLeftElem->v.nType)){ if (pLeftElem->v.i64 == pRightElem->v.i64) { return 0; } else { return pLeftElem->v.i64 > pRightElem->v.i64 ? 1 : -1; } + } else { + if (pLeftElem->v.u64 == pRightElem->v.u64) { + return 0; + } else { + return pLeftElem->v.u64 > pRightElem->v.u64 ? 1 : -1; + } } } @@ -2055,7 +2388,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { switch (type) { case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_INT: { - int32_t *output = (int32_t *)pCtx->aOutputBuf; + int32_t *output = (int32_t *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { *output = (int32_t)tvp[i]->v.i64; } @@ -2063,21 +2396,21 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { } case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_BIGINT: { - int64_t *output = (int64_t *)pCtx->aOutputBuf; + int64_t *output = (int64_t *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { *output = tvp[i]->v.i64; } break; } case TSDB_DATA_TYPE_DOUBLE: { - double *output = (double *)pCtx->aOutputBuf; + double *output = (double *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { *output = tvp[i]->v.dKey; } break; } case TSDB_DATA_TYPE_FLOAT: { - float *output = (float *)pCtx->aOutputBuf; + float *output = (float *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { *output = (float)tvp[i]->v.dKey; } @@ -2085,7 +2418,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { } case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_SMALLINT: { - int16_t *output = (int16_t *)pCtx->aOutputBuf; + int16_t *output = (int16_t *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { *output = (int16_t)tvp[i]->v.i64; } @@ -2093,7 +2426,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { } case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_TINYINT: { - int8_t *output = (int8_t *)pCtx->aOutputBuf; + int8_t *output = (int8_t *)pCtx->pOutput; for (int32_t i = 0; i < len; ++i, output += step) { *output = (int8_t)tvp[i]->v.i64; } @@ -2115,7 +2448,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { // todo check malloc failure char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES); for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { - pData[i] = pCtx->tagInfo.pTagCtxList[i]->aOutputBuf; + pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput; } for (int32_t i = 0; i < len; ++i, output += step) { @@ -2143,13 +2476,13 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { // only the first_stage_merge is directly written data into final output buffer if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { - return (STopBotInfo*) pCtx->aOutputBuf; + return (STopBotInfo*) pCtx->pOutput; } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer return GET_ROWCELL_INTERBUF(pResInfo); } } -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) { +bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; @@ -2164,7 +2497,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha tValuePair **pRes = (tValuePair**) pTopBotInfo->res; - if (functionId == TSDB_FUNC_TOP) { + if (pCtx->functionId == TSDB_FUNC_TOP) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: return GET_INT8_VAL(maxval) > pRes[0]->v.i64; @@ -2235,9 +2568,13 @@ static bool top_bottom_function_setup(SQLFunctionCtx *pCtx) { static void top_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - + STopBotInfo *pRes = getTopBotOutputInfo(pCtx); assert(pRes->num >= 0); + + if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { + buildTopBotStruct(pRes, pCtx); + } for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); @@ -2310,6 +2647,10 @@ static void bottom_function(SQLFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); + if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { + buildTopBotStruct(pRes, pCtx); + } + for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); TSKEY ts = GET_TS_DATA(pCtx, i); @@ -2344,6 +2685,11 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { } STopBotInfo *pRes = getTopBotOutputInfo(pCtx); + + if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) { + buildTopBotStruct(pRes, pCtx); + } + SET_VAL(pCtx, 1, 1); do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64, pData, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); @@ -2424,19 +2770,34 @@ static void percentile_function(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { + // all data are null, set it completed + if (pInfo->numOfElems == 0) { + pResInfo->complete = true; + } else { + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); + } + + pInfo->stage += 1; + } + // the first stage, only acquire the min/max value if (pInfo->stage == 0) { if (pCtx->preAggVals.isSet) { double tmin = 0.0, tmax = 0.0; - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); - tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); - } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); - tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); + if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) { + tmin = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_INT64_VAL(&pCtx->preAggVals.statis.max); + } else if (IS_FLOAT_TYPE(pCtx->inputType)) { + tmin = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.min); + tmax = GET_DOUBLE_VAL(&pCtx->preAggVals.statis.max); + } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) { + tmin = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.min); + tmax = (double)GET_UINT64_VAL(&pCtx->preAggVals.statis.max); } else { assert(true); } + if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) { SET_DOUBLE_VAL(&pInfo->minval, tmin); } @@ -2493,10 +2854,20 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->stage == 0) { + if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) { + // all data are null, set it completed + if (pInfo->numOfElems == 0) { + pResInfo->complete = true; + } else { + pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval); + } + + pInfo->stage += 1; + } + + if (pInfo->stage == 0) { double v = 0; GET_TYPED_DATA(v, double, pCtx->inputType, pData); @@ -2527,9 +2898,9 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { tMemBucket * pMemBucket = ppInfo->pMemBucket; if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null assert(ppInfo->numOfElems == 0); - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); } else { - *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); + *(double *)pCtx->pOutput = getPercentile(pMemBucket, v); } tMemBucketDestroy(pMemBucket); @@ -2565,7 +2936,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SAPercentileInfo* pInfo = NULL; if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) { - pInfo = (SAPercentileInfo*) pCtx->aOutputBuf; + pInfo = (SAPercentileInfo*) pCtx->pOutput; } else { pInfo = GET_ROWCELL_INTERBUF(pResInfo); } @@ -2679,10 +3050,10 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double ratio[] = {v}; double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->aOutputBuf, res, sizeof(double)); + memcpy(pCtx->pOutput, res, sizeof(double)); free(res); } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } } else { @@ -2690,10 +3061,10 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double ratio[] = {v}; double *res = tHistogramUniform(pOutput->pHisto, ratio, 1); - memcpy(pCtx->aOutputBuf, res, sizeof(double)); + memcpy(pCtx->pOutput, res, sizeof(double)); free(res); } else { // no need to free - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } } @@ -2761,17 +3132,17 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { numOfElem++; } break; - }; + } case TSDB_DATA_TYPE_BIGINT: { int64_t *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); break; - }; + } case TSDB_DATA_TYPE_DOUBLE: { double *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); break; - }; + } case TSDB_DATA_TYPE_FLOAT: { float *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); @@ -2781,12 +3152,32 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { int16_t *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); break; - }; + } case TSDB_DATA_TYPE_TINYINT: { int8_t *p = pData; LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); break; - }; + } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *p = pData; + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *p = pData; + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *p = pData; + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t *p = pData; + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].dKey); + break; + } } pInfo->startVal = x; @@ -2859,7 +3250,7 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->num == 0) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } @@ -2878,16 +3269,16 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { param[1][2] /= param[1][1]; int32_t maxOutputSize = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE; - size_t n = snprintf(varDataVal(pCtx->aOutputBuf), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", + size_t n = snprintf(varDataVal(pCtx->pOutput), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}", param[0][2], param[1][2]); - varDataSetLen(pCtx->aOutputBuf, n); + varDataSetLen(pCtx->pOutput, n); doFinalizer(pCtx); } static void date_col_output_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); - *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp; + *(int64_t *)(pCtx->pOutput) = pCtx->startTs; } static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -2904,15 +3295,13 @@ static void col_project_function(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { - memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, + memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, pCtx->inputBytes); } } - - pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -2928,9 +3317,9 @@ static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { INC_INIT_VAL(pCtx, 1); char *pData = GET_INPUT_DATA(pCtx, index); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + memcpy(pCtx->pOutput, pData, pCtx->inputBytes); - pCtx->aOutputBuf += pCtx->inputBytes; + pCtx->pOutput += pCtx->inputBytes; } /** @@ -2943,22 +3332,22 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { assert(pCtx->inputBytes == pCtx->outputBytes); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); - char* data = pCtx->aOutputBuf; - pCtx->aOutputBuf += pCtx->outputBytes; + tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); + char* data = pCtx->pOutput; + pCtx->pOutput += pCtx->outputBytes; // directly copy from the first one for (int32_t i = 1; i < pCtx->size; ++i) { - memmove(pCtx->aOutputBuf, data, pCtx->outputBytes); - pCtx->aOutputBuf += pCtx->outputBytes; + memmove(pCtx->pOutput, data, pCtx->outputBytes); + pCtx->pOutput += pCtx->outputBytes; } } static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { INC_INIT_VAL(pCtx, 1); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->tag.nType, true); - pCtx->aOutputBuf += pCtx->outputBytes; + tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->tag.nType, true); + pCtx->pOutput += pCtx->outputBytes; } /** @@ -2970,19 +3359,19 @@ static void tag_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { */ static void tag_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); + tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); } static void tag_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); - tVariantDump(&pCtx->tag, pCtx->aOutputBuf, pCtx->outputType, true); + tVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true); } static void copy_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); char *pData = GET_INPUT_DATA_LIST(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); + assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); } enum { @@ -3015,7 +3404,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { int32_t *pData = (int32_t *)data; - int32_t *pOutput = (int32_t *)pCtx->aOutputBuf; + int32_t *pOutput = (int32_t *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3047,7 +3436,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { }; case TSDB_DATA_TYPE_BIGINT: { int64_t *pData = (int64_t *)data; - int64_t *pOutput = (int64_t *)pCtx->aOutputBuf; + int64_t *pOutput = (int64_t *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3079,7 +3468,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { } case TSDB_DATA_TYPE_DOUBLE: { double *pData = (double *)data; - double *pOutput = (double *)pCtx->aOutputBuf; + double *pOutput = (double *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3109,7 +3498,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { } case TSDB_DATA_TYPE_FLOAT: { float *pData = (float *)data; - float *pOutput = (float *)pCtx->aOutputBuf; + float *pOutput = (float *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3142,7 +3531,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { } case TSDB_DATA_TYPE_SMALLINT: { int16_t *pData = (int16_t *)data; - int16_t *pOutput = (int16_t *)pCtx->aOutputBuf; + int16_t *pOutput = (int16_t *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { @@ -3173,7 +3562,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { } case TSDB_DATA_TYPE_TINYINT: { int8_t *pData = (int8_t *)data; - int8_t *pOutput = (int8_t *)pCtx->aOutputBuf; + int8_t *pOutput = (int8_t *)pCtx->pOutput; for (; i < pCtx->size && i >= 0; i += step) { if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) { @@ -3218,9 +3607,6 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems; GET_RES_INFO(pCtx)->numOfRes += forwardStep; - - pCtx->aOutputBuf += forwardStep * pCtx->outputBytes; - pCtx->ptsOutputBuf = (char*)pCtx->ptsOutputBuf + forwardStep * TSDB_KEYSIZE; } } @@ -3230,7 +3616,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { (ctx)->param[1].nType = (ctx)->inputType; \ *(type *)&(ctx)->param[1].i64 = *(type *)(d); \ } else { \ - *(type *)(ctx)->aOutputBuf = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64)); \ + *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64)); \ *(type *)(&(ctx)->param[1].i64) = *(type *)(d); \ *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index); \ } \ @@ -3255,7 +3641,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { pCtx->param[1].nType = pCtx->inputType; pCtx->param[1].i64 = *(int32_t *)pData; } else { - *(int32_t *)pCtx->aOutputBuf = *(int32_t *)pData - (int32_t)pCtx->param[1].i64; + *(int32_t *)pCtx->pOutput = *(int32_t *)pData - (int32_t)pCtx->param[1].i64; pCtx->param[1].i64 = *(int32_t *)pData; *(int64_t *)pCtx->ptsOutputBuf = GET_TS_DATA(pCtx, index); } @@ -3286,7 +3672,7 @@ static void diff_function_f(SQLFunctionCtx *pCtx, int32_t index) { } if (GET_RES_INFO(pCtx)->numOfRes > 0) { - pCtx->aOutputBuf += pCtx->outputBytes * step; + pCtx->pOutput += pCtx->outputBytes * step; pCtx->ptsOutputBuf = (char *)pCtx->ptsOutputBuf + TSDB_KEYSIZE * step; } } @@ -3302,7 +3688,7 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { } } - assert(index >= 0 && colId >= 0); + assert(index >= 0 /*&& colId >= 0*/); return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } @@ -3310,10 +3696,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; - arithmeticTreeTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); - - pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size; - pCtx->param[1].pz = NULL; + arithmeticTreeTraverse(sas->pArithExpr->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData); } static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -3321,9 +3704,9 @@ static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz; sas->offset = index; - arithmeticTreeTraverse(sas->pArithExpr->pExpr, 1, pCtx->aOutputBuf, sas, pCtx->order, getArithColumnData); + arithmeticTreeTraverse(sas->pArithExpr->pExpr, 1, pCtx->pOutput, sas, pCtx->order, getArithColumnData); - pCtx->aOutputBuf += pCtx->outputBytes; + pCtx->pOutput += pCtx->outputBytes; } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ @@ -3416,6 +3799,14 @@ static void spread_function(SQLFunctionCtx *pCtx) { LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint8_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint16_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint32_t, pCtx->inputType, numOfElems); + } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) { + LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint64_t, pCtx->inputType, numOfElems); } if (!pCtx->hasNull) { @@ -3432,7 +3823,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } @@ -3475,7 +3866,7 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { pInfo->hasResult = DATA_SET_FLAG; if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } @@ -3511,21 +3902,21 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); if (pResInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } - *(double *)pCtx->aOutputBuf = pCtx->param[3].dKey - pCtx->param[0].dKey; + *(double *)pCtx->pOutput = pCtx->param[3].dKey - pCtx->param[0].dKey; } else { assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); return; } - *(double *)pCtx->aOutputBuf = pInfo->max - pInfo->min; + *(double *)pCtx->pOutput = pInfo->max - pInfo->min; } GET_RES_INFO(pCtx)->numOfRes = 1; // todo add test case @@ -3677,6 +4068,58 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si } break; } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *val = (uint8_t*) GET_INPUT_DATA(pCtx, 0); + for (; i < size && i >= 0; i += step) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *val = (uint16_t*) GET_INPUT_DATA(pCtx, 0); + for (; i < size && i >= 0; i += step) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *val = (uint32_t*) GET_INPUT_DATA(pCtx, 0); + for (; i < size && i >= 0; i += step) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + SPoint1 st = {.key = tsList[i], .val = val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t *val = (uint64_t*) GET_INPUT_DATA(pCtx, 0); + for (; i < size && i >= 0; i += step) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } default: assert(0); } @@ -3715,7 +4158,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { } if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); + memcpy(pCtx->pOutput, pInfo, sizeof(STwaInfo)); } } @@ -3735,7 +4178,7 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { } if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); } } @@ -3748,8 +4191,8 @@ void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); - pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult; + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); + pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { @@ -3757,15 +4200,15 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); if (pInfo->win.ekey == pInfo->win.skey) { - *(double *)pCtx->aOutputBuf = pInfo->p.val; + *(double *)pCtx->pOutput = pInfo->p.val; } else { - *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); + *(double *)pCtx->pOutput = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); } GET_RES_INFO(pCtx)->numOfRes = 1; @@ -3776,89 +4219,111 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { * * @param pCtx */ -static void interp_function(SQLFunctionCtx *pCtx) { - // at this point, the value is existed, return directly - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - assert(pCtx->startOffset == 0); +static void interp_function_impl(SQLFunctionCtx *pCtx) { + int32_t type = (int32_t) pCtx->param[2].i64; + if (type == TSDB_FILL_NONE) { + return; + } - if (pCtx->size == 1) { - char *pData = GET_INPUT_DATA_LIST(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); + if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) { + *(TSKEY *)pCtx->pOutput = pCtx->startTs; + } else if (type == TSDB_FILL_NULL) { + setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes); + } else if (type == TSDB_FILL_SET_VALUE) { + tVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); } else { - /* - * use interpolation to generate the result. - * Note: the result of primary timestamp column uses the timestamp specified by user in the query sql - */ - assert(pCtx->size == 2); - if (pInfo->type == TSDB_FILL_NONE) { // set no output result - return; - } - - if (pInfo->primaryCol == 1) { - *(TSKEY *) pCtx->aOutputBuf = pInfo->ts; - } else { - if (pInfo->type == TSDB_FILL_NULL) { - if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->aOutputBuf, pCtx->outputType); + if (pCtx->start.key != INT64_MIN && pCtx->start.key < pCtx->startTs && pCtx->end.key > pCtx->startTs) { + if (type == TSDB_FILL_PREV) { + if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { + SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val); } else { - setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->outputBytes, pCtx->inputType); } - - SET_VAL(pCtx, pCtx->size, 1); - } else if (pInfo->type == TSDB_FILL_SET_VALUE) { - tVariantDump(&pCtx->param[1], pCtx->aOutputBuf, pCtx->inputType, true); - } else if (pInfo->type == TSDB_FILL_PREV) { - char *data = GET_INPUT_DATA(pCtx, 0); - assignVal(pCtx->aOutputBuf, data, pCtx->outputBytes, pCtx->outputType); - - SET_VAL(pCtx, pCtx->size, 1); - } else if (pInfo->type == TSDB_FILL_LINEAR) { - char *data1 = GET_INPUT_DATA(pCtx, 0); - char *data2 = GET_INPUT_DATA(pCtx, 1); - - TSKEY key1 = pCtx->ptsList[0]; - TSKEY key2 = pCtx->ptsList[1]; - - SPoint point1 = {.key = key1, .val = data1}; - SPoint point2 = {.key = key2, .val = data2}; - - SPoint point = {.key = pInfo->ts, .val = pCtx->aOutputBuf}; - + } else if (type == TSDB_FILL_NEXT) { + if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) { + SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val); + } else { + assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->outputBytes, pCtx->inputType); + } + } else if (type == TSDB_FILL_LINEAR) { + SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val}; + SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val}; + SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; + int32_t srcType = pCtx->inputType; - if ((srcType >= TSDB_DATA_TYPE_TINYINT && srcType <= TSDB_DATA_TYPE_BIGINT) || - srcType == TSDB_DATA_TYPE_TIMESTAMP || srcType == TSDB_DATA_TYPE_DOUBLE) { - point1.val = data1; - point2.val = data2; - - if (isNull(data1, srcType) || isNull(data2, srcType)) { - setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); - } else { - taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point); - } - } else if (srcType == TSDB_DATA_TYPE_FLOAT) { - point1.val = data1; - point2.val = data2; - - if (isNull(data1, srcType) || isNull(data2, srcType)) { - setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); + if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? + if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) { + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } else { - taosGetLinearInterpolationVal(pCtx->outputType, &point1, &point2, &point); + taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); } - } else { - if (srcType == TSDB_DATA_TYPE_BINARY || srcType == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pCtx->aOutputBuf, pCtx->inputType); + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); + } + } + } else { + // no data generated yet + if (pCtx->size == 1) { + return; + } + + // check the timestamp in input buffer + TSKEY skey = GET_TS_DATA(pCtx, 0); + TSKEY ekey = GET_TS_DATA(pCtx, 1); + + // no data generated yet + if (!(skey < pCtx->startTs && ekey > pCtx->startTs)) { + return; + } + + assert(pCtx->start.key == INT64_MIN && skey < pCtx->startTs && ekey > pCtx->startTs); + + if (type == TSDB_FILL_PREV) { + assignVal(pCtx->pOutput, pCtx->pInput, pCtx->outputBytes, pCtx->inputType); + } else if (type == TSDB_FILL_NEXT) { + char* val = ((char*)pCtx->pInput) + pCtx->inputBytes; + assignVal(pCtx->pOutput, val, pCtx->outputBytes, pCtx->inputType); + } else if (type == TSDB_FILL_LINEAR) { + char *start = GET_INPUT_DATA(pCtx, 0); + char *end = GET_INPUT_DATA(pCtx, 1); + + SPoint point1 = {.key = skey, .val = start}; + SPoint point2 = {.key = ekey, .val = end}; + SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput}; + + int32_t srcType = pCtx->inputType; + if (IS_NUMERIC_TYPE(srcType)) { // TODO should find the not null data? + if (isNull(start, srcType) || isNull(end, srcType)) { + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } else { - setNull(pCtx->aOutputBuf, srcType, pCtx->inputBytes); + taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, srcType); } + } else { + setNull(pCtx->pOutput, srcType, pCtx->inputBytes); } } } } - - SET_VAL(pCtx, pCtx->size, 1); + + SET_VAL(pCtx, 1, 1); +} + +static void interp_function(SQLFunctionCtx *pCtx) { + // at this point, the value is existed, return directly + if (pCtx->size > 0) { + // impose the timestamp check + TSKEY key = GET_TS_DATA(pCtx, 0); + if (key == pCtx->startTs) { + char *pData = GET_INPUT_DATA(pCtx, 0); + assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType); + SET_VAL(pCtx, 1, 1); + } else { + interp_function_impl(pCtx); + } + } else { //no qualified data rows and interpolation is required + interp_function_impl(pCtx); + } } static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { @@ -3918,11 +4383,22 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) { STSBuf * pTSbuf = pInfo->pTSBuf; tsBufFlush(pTSbuf); - - *(FILE **)pCtx->aOutputBuf = pTSbuf->f; + qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); + + // TODO refactor transfer ownership of current file + *(FILE **)pCtx->pOutput = pTSbuf->f; + + pResInfo->complete = true; + + // get the file size + struct stat fStat; + if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { + pResInfo->numOfRes = fStat.st_size; + } pTSbuf->remainOpen = true; tsBufDestroy(pTSbuf); + doFinalizer(pCtx); } @@ -3964,7 +4440,7 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->pOutput + pCtx->outputBytes; SRateInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->CorrectionValue = 0; @@ -4033,7 +4509,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4075,7 +4551,7 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4083,10 +4559,10 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); - pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); + pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; - SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; + SRateInfo* pRateInfo = (SRateInfo*)pCtx->pInput; qDebug("%p rate_func_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); } @@ -4099,13 +4575,13 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); + *(double*)pCtx->pOutput = do_calc_rate(pRateInfo); - qDebug("rate_finalizer() output result:%f", *(double *)pCtx->aOutputBuf); + qDebug("rate_finalizer() output result:%f", *(double *)pCtx->pOutput); // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; @@ -4166,7 +4642,7 @@ static void irate_function(SQLFunctionCtx *pCtx) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4200,7 +4676,7 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { // keep the data into the final output buffer for super table query since this execution may be the last one if (pCtx->stableQuery) { - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4232,7 +4708,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { if (DATA_SET_FLAG == pRateInfo->hasResult) { pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, pRateInfo->num, 1); - memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); + memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4248,17 +4724,17 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { qDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } if (pRateInfo->num == 0) { // from meter - *(double*)pCtx->aOutputBuf = do_calc_rate(pRateInfo); + *(double*)pCtx->pOutput = do_calc_rate(pRateInfo); } else if (pCtx->functionId == TSDB_FUNC_SUM_RATE || pCtx->functionId == TSDB_FUNC_SUM_IRATE) { - *(double*)pCtx->aOutputBuf = pRateInfo->sum; + *(double*)pCtx->pOutput = pRateInfo->sum; } else { - *(double*)pCtx->aOutputBuf = pRateInfo->sum / pRateInfo->num; + *(double*)pCtx->pOutput = pRateInfo->sum / pRateInfo->num; } pResInfo->numOfRes = 1; @@ -4266,10 +4742,126 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } +void blockInfo_func(SQLFunctionCtx* pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); -///////////////////////////////////////////////////////////////////////////////////////////// + int32_t len = *(int32_t*) pCtx->pInput; + blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); + pDist->rowSize = (int16_t) pCtx->param[0].i64; + memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); + pResInfo->numOfRes = 1; + pResInfo->hasResult = DATA_SET_FLAG; +} + +static void mergeTableBlockDist(STableBlockDist* pDist, const STableBlockDist* pSrc) { + assert(pDist != NULL && pSrc != NULL); + pDist->numOfTables += pSrc->numOfTables; + pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable; + pDist->numOfFiles += pSrc->numOfFiles; + pDist->totalSize += pSrc->totalSize; + + if (pDist->dataBlockInfos == NULL) { + pDist->dataBlockInfos = taosArrayInit(4, sizeof(SFileBlockInfo)); + } + + taosArrayPushBatch(pDist->dataBlockInfos, pSrc->dataBlockInfos->pData, (int32_t) taosArrayGetSize(pSrc->dataBlockInfos)); +} + +void block_func_merge(SQLFunctionCtx* pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); + STableBlockDist info = {0}; + + int32_t len = *(int32_t*) pCtx->pInput; + blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); + + mergeTableBlockDist(pDist, &info); +} + +static int32_t doGetPercentile(const SArray* pArray, double rate) { + int32_t len = (int32_t)taosArrayGetSize(pArray); + if (len <= 0) { + return 0; + } + + assert(rate >= 0 && rate <= 1.0); + int idx = (int32_t)((len - 1) * rate); + + return ((SFileBlockInfo *)(taosArrayGet(pArray, idx)))->numOfRows; +} + +static int compareBlockInfo(const void *pLeft, const void *pRight) { + int32_t left = ((SFileBlockInfo *)pLeft)->numOfRows; + int32_t right = ((SFileBlockInfo *)pRight)->numOfRows; + + if (left > right) return 1; + if (left < right) return -1; + return 0; +} + +void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { + if (pTableBlockDist == NULL) { + return; + } + + int64_t min = INT64_MAX, max = INT64_MIN, avg = 0; + SArray* blockInfos= pTableBlockDist->dataBlockInfos; + int64_t totalRows = 0, totalBlocks = taosArrayGetSize(blockInfos); + + for (size_t i = 0; i < taosArrayGetSize(blockInfos); i++) { + SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i); + int64_t rows = blockInfo->numOfRows; + + min = MIN(min, rows); + max = MAX(max, rows); + totalRows += rows; + } + + avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0; + taosArraySort(blockInfos, compareBlockInfo); + + uint64_t totalLen = pTableBlockDist->totalSize; + int32_t rowSize = pTableBlockDist->rowSize; + + int sz = sprintf(result + VARSTR_HEADER_SIZE, + "summary: \n\t " + "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t " + "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t " + "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t " + "Rows=[%"PRId64"], Blocks=[%"PRId64"], Size=[%.3f(Kb)] Comp=[%.2f%%]\n\t " + "RowsInMem=[%d] \n\t SeekHeaderTime=[%d(us)]", + doGetPercentile(blockInfos, 0.05), doGetPercentile(blockInfos, 0.10), + doGetPercentile(blockInfos, 0.20), doGetPercentile(blockInfos, 0.30), + doGetPercentile(blockInfos, 0.40), doGetPercentile(blockInfos, 0.50), + doGetPercentile(blockInfos, 0.60), doGetPercentile(blockInfos, 0.70), + doGetPercentile(blockInfos, 0.80), doGetPercentile(blockInfos, 0.90), + doGetPercentile(blockInfos, 0.95), doGetPercentile(blockInfos, 0.99), + min, max, avg, 0.0, + totalRows, totalBlocks, totalLen/1024.0, (double)(totalLen*100.0)/(rowSize*totalRows), + pTableBlockDist->numOfRowsInMemTable, pTableBlockDist->firstSeekTimeUs); + varDataSetLen(result, sz); + UNUSED(sz); +} + +void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); + + pDist->rowSize = (int16_t)pCtx->param[0].i64; + generateBlockDistResult(pDist, pCtx->pOutput); + + // cannot set the numOfIteratedElems again since it is set during previous iteration + pResInfo->numOfRes = 1; + pResInfo->hasResult = DATA_SET_FLAG; + + doFinalizer(pCtx); +} + +///////////////////////////////////////////////////////////////////////////////////////////// /* * function compatible list. * tag and ts are not involved in the compatibility check @@ -4288,11 +4880,11 @@ int32_t functionCompatList[] = { 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, - // sum_rate, sum_irate, avg_rate, avg_irate - 1, 1, 1, 1, + // sum_rate, sum_irate, avg_rate, avg_irate, tid_tag, blk_info + 1, 1, 1, 1, 6, 7 }; -SQLAggFuncElem aAggs[] = {{ +SAggFunctionInfo aAggs[] = {{ // 0, count function does not invoke the finalize function "count", TSDB_FUNC_COUNT, @@ -4304,7 +4896,7 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, count_func_merge, - count_load_data_info, + countRequired, }, { // 1 @@ -4366,7 +4958,7 @@ SQLAggFuncElem aAggs[] = {{ // 5 "stddev", TSDB_FUNC_STDDEV, - TSDB_FUNC_INVALID_ID, + TSDB_FUNC_STDDEV_DST, TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STREAM | TSDB_FUNCSTATE_OF, function_setup, stddev_function, @@ -4489,7 +5081,7 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, spread_function_finalizer, spread_func_merge, - count_load_data_info, + countRequired, }, { // 14 @@ -4531,7 +5123,7 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - no_data_info, + noDataRequired, }, { // 17 @@ -4559,7 +5151,7 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - no_data_info, + noDataRequired, }, { // 19 @@ -4587,7 +5179,7 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - no_data_info, + noDataRequired, }, { // 21, column project sql function @@ -4615,7 +5207,7 @@ SQLAggFuncElem aAggs[] = {{ no_next_step, doFinalizer, copy_function, - no_data_info, + noDataRequired, }, { // 23 @@ -4676,6 +5268,20 @@ SQLAggFuncElem aAggs[] = {{ }, { // 27 + "stddev", // return table id and the corresponding tags for join match and subscribe + TSDB_FUNC_STDDEV_DST, + TSDB_FUNC_AVG, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, + function_setup, + stddev_dst_function, + stddev_dst_function_f, + no_next_step, + stddev_dst_finalizer, + stddev_dst_merge, + dataBlockRequired, + }, + { + // 28 "interp", TSDB_FUNC_INTERP, TSDB_FUNC_INTERP, @@ -4689,7 +5295,7 @@ SQLAggFuncElem aAggs[] = {{ dataBlockRequired, }, { - // 28 + // 29 "rate", TSDB_FUNC_RATE, TSDB_FUNC_RATE, @@ -4703,7 +5309,7 @@ SQLAggFuncElem aAggs[] = {{ dataBlockRequired, }, { - // 29 + // 30 "irate", TSDB_FUNC_IRATE, TSDB_FUNC_IRATE, @@ -4717,7 +5323,7 @@ SQLAggFuncElem aAggs[] = {{ dataBlockRequired, }, { - // 30 + // 31 "sum_rate", TSDB_FUNC_SUM_RATE, TSDB_FUNC_SUM_RATE, @@ -4731,7 +5337,7 @@ SQLAggFuncElem aAggs[] = {{ dataBlockRequired, }, { - // 31 + // 32 "sum_irate", TSDB_FUNC_SUM_IRATE, TSDB_FUNC_SUM_IRATE, @@ -4745,7 +5351,7 @@ SQLAggFuncElem aAggs[] = {{ dataBlockRequired, }, { - // 32 + // 33 "avg_rate", TSDB_FUNC_AVG_RATE, TSDB_FUNC_AVG_RATE, @@ -4759,7 +5365,7 @@ SQLAggFuncElem aAggs[] = {{ dataBlockRequired, }, { - // 33 + // 34 "avg_irate", TSDB_FUNC_AVG_IRATE, TSDB_FUNC_AVG_IRATE, @@ -4773,8 +5379,8 @@ SQLAggFuncElem aAggs[] = {{ dataBlockRequired, }, { - // 34 - "tid_tag", // return table id and the corresponding tags for join match and subscribe + // 35 + "tbid", // return table id and the corresponding tags for join match and subscribe TSDB_FUNC_TID_TAG, TSDB_FUNC_TID_TAG, TSDB_FUNCSTATE_MO | TSDB_FUNCSTATE_STABLE, @@ -4785,4 +5391,18 @@ SQLAggFuncElem aAggs[] = {{ noop1, noop1, dataBlockRequired, + }, + { + // 35 + "_block_dist", // return table id and the corresponding tags for join match and subscribe + TSDB_FUNC_BLKINFO, + TSDB_FUNC_BLKINFO, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, + function_setup, + blockInfo_func, + noop2, + no_next_step, + blockinfo_func_finalizer, + block_func_merge, + dataBlockRequired, }}; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 90cd74de72f6b47b7bf99b6055d4a4c5e9cfa7ab..234bb81e2bb8bc86a46390f4bc003c41a8e98676 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -15,39 +15,33 @@ #include "os.h" #include "qFill.h" #include "taosmsg.h" -#include "tcache.h" #include "tglobal.h" #include "exception.h" #include "hash.h" -#include "qAst.h" +#include "texpr.h" #include "qExecutor.h" #include "qResultbuf.h" #include "qUtil.h" -#include "query.h" #include "queryLog.h" #include "tlosertree.h" #include "ttype.h" - -#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1) - -/** - * check if the primary column is load by default, otherwise, the program will - * forced to load primary column explicitly. - */ -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) -#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) +#include "tscompression.h" #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) #define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN) #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) -#define GET_QINFO_ADDR(x) ((SQInfo *)((char *)(x)-offsetof(SQInfo, runtimeEnv))) - -#define GET_COL_DATA_POS(query, index, step) ((query)->pos + (index) * (step)) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) +#define CHECK_IF_QUERY_KILLED(_q) \ + do { \ + if (isQueryKilled((_q)->qinfo)) { \ + longjmp((_q)->env, TSDB_CODE_TSC_QUERY_CANCELLED); \ + } \ + } while (0) + #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} #define TIME_WINDOW_COPY(_dst, _src) do {\ @@ -55,40 +49,16 @@ (_dst).ekey = (_src).ekey;\ } while (0) -enum { - // when query starts to execute, this status will set - QUERY_NOT_COMPLETED = 0x1u, - - /* result output buffer is full, current query is paused. - * this status is only exist in group-by clause and diff/add/division/multiply/ query. - */ - QUERY_RESBUF_FULL = 0x2u, - - /* query is over - * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. - * 2. when all data within queried time window, it is also denoted as query_completed - */ - QUERY_COMPLETED = 0x4u, - - /* when the result is not completed return to client, this status will be - * usually used in case of interval query with interpolation option - */ - QUERY_OVER = 0x8u, -}; - enum { TS_JOIN_TS_EQUAL = 0, TS_JOIN_TS_NOT_EQUALS = 1, TS_JOIN_TAG_NOT_EQUALS = 2, }; -typedef struct { - int32_t status; // query status - TSKEY lastKey; // the lastKey value before query executed - STimeWindow w; // whole query time window - int32_t windowIndex; // index of active time window result for interval query - STSCursor cur; -} SQueryStatusInfo; +typedef enum SResultTsInterpType { + RESULT_ROW_START_INTERP = 1, + RESULT_ROW_END_INTERP = 2, +} SResultTsInterpType; #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { @@ -126,13 +96,12 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) #define GET_NUM_OF_TABLEGROUP(q) taosArrayGetSize((q)->tableqinfoGroupInfo.pGroupList) -#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) #define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0) -static void setQueryStatus(SQuery *pQuery, int8_t status); -static void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv); -static int32_t getMaximumIdleDurationSec() { +uint64_t queryHandleId = 0; + +int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } @@ -173,92 +142,124 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { tw->ekey -= 1; } -#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) -#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables)) +static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); +static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfCols, int32_t* rowCellInfoOffset); -// todo move to utility -static int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo); - -static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); -static void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); -static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId); +static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex); -static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); -static void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); -static void buildTagQueryResult(SQInfo *pQInfo); -static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t checkForQueryBuf(size_t numOfTables); +static int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); -static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win); -static STableIdInfo createTableIdInfo(SQuery* pQuery); - -bool doFilterData(SQuery *pQuery, int32_t elemPos) { - for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - - char *pElem = (char*)pFilterInfo->pData + pFilterInfo->info.bytes * elemPos; +static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo); + +static void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream); + +static int32_t getNumOfScanTimes(SQuery* pQuery); +static bool isFixedOutputQuery(SQuery* pQuery); + +static SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); +static SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); +static SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); + +static SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); +static SOperatorInfo* createOffsetOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); +static SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); +static SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); + +static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); +static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); +static void destroyArithOperatorInfo(void* param, int32_t numOfOutput); +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); +static void destroyOperatorInfo(SOperatorInfo* pOperator); + +static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); + +static int32_t getGroupbyColumnIndex(SSqlGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SGroupbyOperatorInfo *pInfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); + +static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); +static void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); +static bool isPointInterpoQuery(SQuery *pQuery); +static void setResultBufSize(SQuery* pQuery, SRspResultInfo* pResultInfo); +static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); +static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, + SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, + int32_t groupIndex); + +// setup the output buffer for each operator +static SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows) { + const static int32_t minSize = 8; + + SSDataBlock *res = calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = numOfOutput; + + res->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData idata = {{0}}; + idata.info.type = pExpr[i].type; + idata.info.bytes = pExpr[i].bytes; + idata.info.colId = pExpr[i].base.resColId; - bool qualified = false; - for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) { - SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j]; + idata.pData = calloc(1, MAX(idata.info.bytes * numOfRows, minSize)); // at least to hold a pointer on x64 platform + taosArrayPush(res->pDataBlock, &idata); + } - bool isnull = isNull(pElem, pFilterInfo->info.type); - if (isnull) { - if (pFilterElem->fp == isNullOperator) { - qualified = true; - break; - } else { - continue; - } - } else { - if (pFilterElem->fp == notNullOperator) { - qualified = true; - break; - } else if (pFilterElem->fp == isNullOperator) { - continue; - } - } + return res; +} - if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo->info.type)) { - qualified = true; - break; - } - } +static void* destroyOutputBuf(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return NULL; + } - if (!qualified) { - return false; - } + int32_t numOfOutput = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + tfree(pColInfoData->pData); } - return true; + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockStatis); + tfree(pBlock); + return NULL; } -int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { +int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput) { SQuery *pQuery = pRuntimeEnv->pQuery; bool hasMainFunction = hasMainOutput(pQuery); - int64_t maxOutput = 0; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pExpr1[j].base.functionId; + int32_t maxOutput = 0; + for (int32_t j = 0; j < numOfOutput; ++j) { + int32_t id = pCtx[j].functionId; /* * ts, tag, tagprj function can not decide the output number of current query * the number of output result is decided by main output */ - if (hasMainFunction && - (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ)) { + if (hasMainFunction && (id == TSDB_FUNC_TS || id == TSDB_FUNC_TAG || id == TSDB_FUNC_TAGPRJ)) { continue; } - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pCtx[j]); if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { maxOutput = pResInfo->numOfRes; } @@ -268,32 +269,14 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { return maxOutput; } -/* - * the value of number of result needs to be update due to offset value upated. - */ -void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - - int16_t functionId = pRuntimeEnv->pCtx[j].functionId; - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || - functionId == TSDB_FUNC_TS_DUMMY) { - continue; - } - - assert(pResInfo->numOfRes > numOfRes); - pResInfo->numOfRes = numOfRes; +static void clearNumOfRes(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t j = 0; j < numOfOutput; ++j) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pCtx[j]); + pResInfo->numOfRes = 0; } } -static UNUSED_FUNC int32_t getMergeResultGroupId(int32_t groupIndex) { - int32_t base = 50000000; - return base + (groupIndex * 10000); -} - -bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { +static bool isGroupbyColumn(SSqlGroupbyExpr *pGroupbyExpr) { if (pGroupbyExpr == NULL || pGroupbyExpr->numOfGroupCols == 0) { return false; } @@ -313,54 +296,37 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { return false; } -int16_t getGroupbyColumnType(SQuery *pQuery, SSqlGroupbyExpr *pGroupbyExpr) { - assert(pGroupbyExpr != NULL); - - int32_t colId = -2; - int16_t type = TSDB_DATA_TYPE_NULL; - - for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { - SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); - if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - colId = pColIndex->colId; - break; - } - } - - for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (colId == pQuery->colList[i].colId) { - type = pQuery->colList[i].type; - break; +static bool isStabledev(SQuery* pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functId = pQuery->pExpr1[i].base.functionId; + if (functId == TSDB_FUNC_STDDEV_DST) { + return true; } } - return type; + return false; } -bool isSelectivityWithTagsQuery(SQuery *pQuery) { +static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { bool hasTags = false; int32_t numOfSelectivity = 0; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functId = pQuery->pExpr1[i].base.functionId; + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functId = pCtx[i].functionId; if (functId == TSDB_FUNC_TAG_DUMMY || functId == TSDB_FUNC_TS_DUMMY) { hasTags = true; continue; } - if ((aAggs[functId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + if ((aAggs[functId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { numOfSelectivity++; } } - if (numOfSelectivity > 0 && hasTags) { - return true; - } - - return false; + return (numOfSelectivity > 0 && hasTags); } -bool isProjQuery(SQuery *pQuery) { +static bool isProjQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functId = pQuery->pExpr1[i].base.functionId; if (functId != TSDB_FUNC_PRJ && functId != TSDB_FUNC_TAGPRJ) { @@ -371,24 +337,7 @@ bool isProjQuery(SQuery *pQuery) { return true; } -bool isTSCompQuery(SQuery *pQuery) { return pQuery->pExpr1[0].base.functionId == TSDB_FUNC_TS_COMP; } - -static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - SQuery* pQuery = pRuntimeEnv->pQuery; - - if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { - pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; - - qDebug("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64, - pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); - assert(pQuery->rec.rows >= 0); - setQueryStatus(pQuery, QUERY_COMPLETED); - return true; - } - - return false; -} +static bool isTsCompQuery(SQuery *pQuery) { return pQuery->pExpr1[0].base.functionId == TSDB_FUNC_TS_COMP; } static bool isTopBottomQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { @@ -408,7 +357,7 @@ static bool isTopBottomQuery(SQuery *pQuery) { static bool timeWindowInterpoRequired(SQuery *pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TWA) { + if (functionId == TSDB_FUNC_TWA || functionId == TSDB_FUNC_INTERP) { return true; } } @@ -434,33 +383,44 @@ static bool hasTagValOutput(SQuery* pQuery) { return false; } -/** - * @param pQuery - * @param col - * @param pDataBlockInfo - * @param pStatis - * @param pColStatis - * @return - */ -static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (pStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { - *pColStatis = &pStatis[pColIndex->colIndex]; - assert((*pColStatis)->colId == pColIndex->colId); - } else { - *pColStatis = NULL; - } - +static bool hasNullRv(SColIndex* pColIndex, SDataStatis *pStatis) { if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } - if ((*pColStatis) != NULL && (*pColStatis)->numOfNull == 0) { + if (pStatis != NULL && pStatis->numOfNull == 0) { return false; } return true; } +static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, SQueryRuntimeEnv* pRuntimeEnv) { + // more than the capacity, reallocate the resources + if (pResultRowInfo->size < pResultRowInfo->capacity) { + return; + } + + int64_t newCapacity = 0; + if (pResultRowInfo->capacity > 10000) { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); + } else { + newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); + } + + char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + if (t == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pResultRowInfo->pResult = (SResultRow **)t; + + int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; + memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); + + pResultRowInfo->capacity = (int32_t)newCapacity; +} + static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { bool existed = false; @@ -491,28 +451,7 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes } if (!existed) { - // TODO refactor - // more than the capacity, reallocate the resources - if (pResultRowInfo->size >= pResultRowInfo->capacity) { - int64_t newCapacity = 0; - if (pResultRowInfo->capacity > 10000) { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); - } else { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); - } - - char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); - if (t == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pResultRowInfo->pResult = (SResultRow **)t; - - int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; - memset(&pResultRowInfo->pResult[pResultRowInfo->capacity], 0, POINTER_BYTES * inc); - - pResultRowInfo->capacity = (int32_t)newCapacity; - } + prepareResultListBuffer(pResultRowInfo, pRuntimeEnv); SResultRow *pResult = NULL; @@ -541,20 +480,50 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes return getResultRow(pResultRowInfo, pResultRowInfo->curIndex); } +static void getInitialStartTimeWindow(SQuery* pQuery, TSKEY ts, STimeWindow* w) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + getAlignQueryTimeWindow(pQuery, ts, ts, pQuery->window.ekey, w); + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pQuery, ts, pQuery->window.ekey, ts, w); + + int64_t key = w->skey; + while(key < ts) { // moving towards end + if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { + key = taosTimeAdd(key, pQuery->interval.sliding, pQuery->interval.slidingUnit, pQuery->precision); + } else { + key += pQuery->interval.sliding; + } + + if (key >= ts) { + break; + } + + w->skey = key; + } + } +} + // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) { +static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SQuery *pQuery) { STimeWindow w = {0}; - if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value - w.skey = pWindowResInfo->prevSKey; + if (pResultRowInfo->curIndex == -1) { // the first window, from the previous stored value + if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { + getInitialStartTimeWindow(pQuery, ts, &w); + pResultRowInfo->prevSKey = w.skey; + } else { + w.skey = pResultRowInfo->prevSKey; + } + if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') { w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1; } else { w.ekey = w.skey + pQuery->interval.interval - 1; } } else { - int32_t slot = curTimeWindowIndex(pWindowResInfo); - SResultRow* pWindowRes = getResultRow(pWindowResInfo, slot); + int32_t slot = curTimeWindowIndex(pResultRowInfo); + SResultRow* pWindowRes = getResultRow(pResultRowInfo, slot); w = pWindowRes->win; } @@ -590,8 +559,8 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo *pWindowResInfo, int64_t t return w; } -static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, - int32_t numOfRowsPerPage) { +// a new buffer page for each table. Needs to opt this design +static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, uint32_t size) { if (pWindowRes->pageId != -1) { return 0; } @@ -609,7 +578,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf pData = getResBufPage(pResultBuf, pi->pageId); pageId = pi->pageId; - if (pData->num >= numOfRowsPerPage) { + if (pData->num + size > pResultBuf->pageSize) { // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); pData = getNewDataBuf(pResultBuf, tid, &pageId); @@ -626,8 +595,9 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf // set the number of rows in current disk page if (pWindowRes->pageId == -1) { // not allocated yet, allocate new buffer pWindowRes->pageId = pageId; - pWindowRes->rowId = (int32_t)(pData->num++); + pWindowRes->offset = (int32_t)pData->num; + pData->num += size; assert(pWindowRes->pageId >= 0); } @@ -635,7 +605,8 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf } static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, - bool masterscan, SResultRow** pResult, int64_t groupId) { + bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -647,7 +618,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRow // not assign result buffer yet, add new result buffer if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->pQuery->intermediateResultRowSize); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -656,21 +627,11 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRow // set time window for current result pResultRow->win = (*win); *pResult = pResultRow; - setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow); + setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); return TSDB_CODE_SUCCESS; } -static bool getResultRowStatus(SResultRowInfo *pWindowResInfo, int32_t slot) { - assert(slot >= 0 && slot < pWindowResInfo->size); - return pWindowResInfo->pResult[slot]->closed; -} - -typedef enum SResultTsInterpType { - RESULT_ROW_START_INTERP = 1, - RESULT_ROW_END_INTERP = 2, -} SResultTsInterpType; - static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { @@ -768,13 +729,14 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } } -static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery, bool timeWindowInterpo) { - if ((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey && (!ascQuery))) { +static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, SQuery* pQuery, TSKEY lastKey) { + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + if ((lastKey > pQuery->window.ekey && ascQuery) || (lastKey < pQuery->window.ekey && (!ascQuery))) { closeAllResultRows(pResultRowInfo); pResultRowInfo->curIndex = pResultRowInfo->size - 1; } else { - int32_t step = ascQuery? 1:-1; - doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey - step, ascQuery, timeWindowInterpo); + int32_t step = ascQuery ? 1 : -1; + doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQuery->timeWindowInterpo); } } @@ -818,18 +780,27 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) { - SQuery *pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - +static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, + int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { + SQuery *pQuery = pRuntimeEnv->pQuery; bool hasPrev = pCtx[0].preAggVals.isSet; - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; + for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].size = forwardStep; - pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); + pCtx[k].startTs = pWin->skey; + + char* start = pCtx[k].pInput; - int32_t functionId = pQuery->pExpr1[k].base.functionId; + int32_t pos = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1); + if (pCtx[k].pInput != NULL) { + pCtx[k].pInput = (char *)pCtx[k].pInput + pos * pCtx[k].inputBytes; + } + + if (tsCol != NULL) { + pCtx[k].ptsList = &tsCol[pos]; + } + + int32_t functionId = pCtx[k].functionId; // not a whole block involved in query processing, statistics data can not be used // NOTE: the original value of isSet have been changed here @@ -843,27 +814,13 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow // restore it pCtx[k].preAggVals.isSet = hasPrev; + pCtx[k].pInput = start; } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset) { - SQuery *pQuery = pRuntimeEnv->pQuery; - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pCtx[k].nStartQueryTimestamp = pWin->skey; - - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pCtx[k], offset); - } - } -} -static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, +static int32_t getNextQualifiedWindow(SQuery* pQuery, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo, TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { - SQuery *pQuery = pRuntimeEnv->pQuery; - getNextTimeWindow(pQuery, pNext); // next time window is not in current block @@ -953,69 +910,6 @@ static FORCE_INLINE TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) return ekey; } -//todo binary search -static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { - int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *p = taosArrayGet(pDataBlock, i); - if (colId == p->info.colId) { - return p->pData; - } - } - - return NULL; -} - -static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas, int32_t col, int32_t size, SArray *pDataBlock) { - if (pDataBlock == NULL) { - return NULL; - } - - char *dataBlock = NULL; - SQuery *pQuery = pRuntimeEnv->pQuery; - - int32_t functionId = pQuery->pExpr1[col].base.functionId; - if (functionId == TSDB_FUNC_ARITHM) { - sas->pArithExpr = &pQuery->pExpr1[col]; - sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); - sas->colList = pQuery->colList; - sas->numOfCols = pQuery->numOfCols; - - // here the pQuery->colList and sas->colList are identical - int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - SColumnInfo *pColMsg = &pQuery->colList[i]; - - dataBlock = NULL; - for (int32_t k = 0; k < numOfCols; ++k) { //todo refactor - SColumnInfoData *p = taosArrayGet(pDataBlock, k); - if (pColMsg->colId == p->info.colId) { - dataBlock = p->pData; - break; - } - } - - assert(dataBlock != NULL); - sas->data[i] = dataBlock; // start from the offset - } - - } else { // other type of query function - SColIndex *pCol = &pQuery->pExpr1[col].base.colInfo; - if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { - SColIndex* pColIndex = &pQuery->pExpr1[col].base.colInfo; - SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex); - assert(p->info.colId == pColIndex->colId); - - dataBlock = p->pData; - } else { - dataBlock = NULL; - } - } - - return dataBlock; -} - static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { if (type == RESULT_ROW_START_INTERP) { for (int32_t k = 0; k < numOfOutput; ++k) { @@ -1029,59 +923,7 @@ static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, in } // window start key interpolation -static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - TSKEY curTs = tsCols[pos]; - TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; - - // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. - // start exactly from this point, no need to do interpolation - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; - if (key == curTs) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - if (lastTs == INT64_MIN && ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - return true; - } - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - TSKEY prevTs = ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))? - lastTs:tsCols[pos - step]; - - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); - return true; -} - -static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t endRowIndex, SArray* pDataBlock, TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { - SQuery* pQuery = pRuntimeEnv->pQuery; - TSKEY actualEndKey = tsCols[endRowIndex]; - - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; - - // not ended in current data block, do not invoke interpolation - if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - return false; - } - - // there is actual end point of current time window, no interpolation need - if (key == actualEndKey) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - return true; - } - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - int32_t nextRowIndex = endRowIndex + step; - assert(nextRowIndex >= 0); - TSKEY nextKey = tsCols[nextRowIndex]; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, RESULT_ROW_END_INTERP); - return true; -} static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, int32_t rowIndex) { @@ -1096,200 +938,502 @@ static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* } } -static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY* tsCols, int32_t step) { +static TSKEY getStartTsKey(SQuery* pQuery, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { TSKEY ts = TSKEY_INITIAL_VAL; + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); if (tsCols == NULL) { - ts = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.skey : pDataBlockInfo->window.ekey; + ts = ascQuery? win->skey : win->ekey; } else { - int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); + int32_t offset = ascQuery? 0:rows-1; ts = tsCols[offset]; } return ts; } -static void doWindowBorderInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray *pDataBlock, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { - if (!pRuntimeEnv->timeWindowInterpo) { - return; +static void setArithParams(SArithmeticSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) { + sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols; + sas->pArithExpr = pExprInfo; + + sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo)); + for(int32_t i = 0; i < sas->numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i); + sas->colList[i] = pColData->info; } - assert(pDataBlock != NULL); + sas->data = calloc(sas->numOfCols, POINTER_BYTES); - SQuery* pQuery = pRuntimeEnv->pQuery; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + // set the input column data + for (int32_t f = 0; f < pSDataBlock->info.numOfCols; ++f) { + SColumnInfoData *pColumnInfoData = taosArrayGet(pSDataBlock->pDataBlock, f); + sas->data[f] = pColumnInfoData->pData; + } +} - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); +static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; - TSKEY *tsCols = (TSKEY *)(pColInfo->pData); - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); } +} - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); - if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1) * step; - - TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); +static void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + if (pCtx[0].functionId == TSDB_FUNC_ARITHM) { + SArithmeticSupport* pSupport = (SArithmeticSupport*) pCtx[0].param[1].pz; + if (pSupport->colList == NULL) { + doSetInputDataBlock(pOperator, pCtx, pBlock, order); + } else { + doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + if (pCtx[0].pInput == NULL && pBlock->pDataBlock != NULL) { + doSetInputDataBlock(pOperator, pCtx, pBlock, order); + } else { + doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); + } } } -/** - * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions - * @param pRuntimeEnv - * @param forwardStep - * @param tsCols - * @param pFields - * @param isDiskFileBlock - * @return the incremental number of output value, so it maybe 0 for fixed number of query, - * such as count/min/max etc. - */ -static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SResultRowInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - - SQuery *pQuery = pRuntimeEnv->pQuery; - int64_t groupId = pQuery->current->groupIndex; +static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].currentStage = (uint8_t)pOperator->pRuntimeEnv->scanFlag; - TSKEY *tsCols = NULL; - if (pDataBlock != NULL) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); - tsCols = (TSKEY *)(pColInfo->pData); - } + setBlockStatisInfo(&pCtx[i], pBlock, &pOperator->pExpr[i].base.colInfo); - SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pRuntimeEnv->sasArray[k], k, pQInfo->vgId); + if (pCtx[i].functionId == TSDB_FUNC_ARITHM) { + setArithParams((SArithmeticSupport*)pCtx[i].param[1].pz, &pOperator->pExpr[i], pBlock); + } else { + SColIndex* pCol = &pOperator->pExpr[i].base.colInfo; + if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || pCol->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; + SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pColIndex->colIndex); + + // in case of the block distribution query, the inputBytes is not a constant value. + pCtx[i].pInput = p->pData; + assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type);// && pCtx[i].inputBytes == p->info.bytes); + + uint32_t status = aAggs[pCtx[i].functionId].status; + if ((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) { + SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); + pCtx[i].ptsList = (int64_t*) tsInfo->pData; + } + } + } } +} - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - int32_t prevIndex = curTimeWindowIndex(pWindowResInfo); - - TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); - STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); +static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - goto _end; + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + pCtx[k].startTs = startTs;// this can be set during create the struct + aAggs[functionId].xFunction(&pCtx[k]); } + } +} + +static void arithmeticApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { + SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t forwardStep = 0; - int32_t startPos = pQuery->pos; + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].startTs = pQuery->window.skey; + aAggs[pCtx[k].functionId].xFunction(&pCtx[k]); + } +} - TSKEY ekey = reviseWindowEkey(pQuery, &win); - forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); +void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, + int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SExprInfo* pExpr = pOperator->pExpr; - // prev time window not interpolation yet. - int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { - for(int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow *pRes = pWindowResInfo->pResult[j]; - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); - continue; - } + SQLFunctionCtx* pCtx = pInfo->pCtx; - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &w, masterScan, &pResult, groupId); - assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + int32_t functionId = pCtx[k].functionId; + if (functionId != TSDB_FUNC_TWA && functionId != TSDB_FUNC_INTERP) { + pCtx[k].start.key = INT64_MIN; + continue; + } - int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + SColIndex * pColIndex = &pExpr[k].base.colInfo; + int16_t index = pColIndex->colIndex; + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, index); - doBlockwiseApplyFunctions(pRuntimeEnv, &w, startPos, 0, tsCols, pDataBlockInfo->rows); - } + assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; - // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); - assert (ret == TSDB_CODE_SUCCESS); + if (prevRowIndex == -1) { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); } - // window start key interpolation - doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &win, pQuery->pos, forwardStep); - doBlockwiseApplyFunctions(pRuntimeEnv, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn, prevEndPos); - if (startPos < 0) { - break; + if (functionId == TSDB_FUNC_INTERP) { + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = prevTs; + pCtx[k].start.val = v1; + + pCtx[k].end.key = curTs; + pCtx[k].end.val = v2; } + } else if (functionId == TSDB_FUNC_TWA) { + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v }; - // null data, failed to allocate more memory buffer - int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - break; + taosGetLinearInterpolationVal(&point, TSDB_DATA_TYPE_DOUBLE, &point1, &point2, TSDB_DATA_TYPE_DOUBLE); + + if (type == RESULT_ROW_START_INTERP) { + pCtx[k].start.key = point.key; + pCtx[k].start.val = v; + } else { + pCtx[k].end.key = point.key; + pCtx[k].end.val = v; } + } + } +} + +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + + TSKEY curTs = tsCols[pos]; + TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; + + // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. + // start exactly from this point, no need to do interpolation + TSKEY key = ascQuery? win->skey:win->ekey; + if (key == curTs) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))? lastTs:tsCols[pos - step]; + + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, + key, RESULT_ROW_START_INTERP); + return true; +} + +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, + int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { + SQueryRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + + TSKEY actualEndKey = tsCols[endRowIndex]; + + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; + + // not ended in current data block, do not invoke interpolation + if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return false; + } + + // there is actual end point of current time window, no interpolation need + if (key == actualEndKey) { + setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + int32_t nextRowIndex = endRowIndex + step; + assert(nextRowIndex >= 0); + + TSKEY nextKey = tsCols[nextRowIndex]; + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, + nextRowIndex, key, RESULT_ROW_END_INTERP); + return true; +} + +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; + if (!pQuery->timeWindowInterpo) { + return; + } + + assert(pBlock != NULL); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - ekey = reviseWindowEkey(pQuery, &nextWin); - forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); - // window start(end) key interpolation - doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &nextWin, startPos, forwardStep); - doBlockwiseApplyFunctions(pRuntimeEnv, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + TSKEY *tsCols = (TSKEY *)(pColInfo->pData); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { // it is not interpolated, now start to generated the interpolated value + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, pBlock->pDataBlock, + tsCols, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } + } else { + setNotInterpoWindowKey(pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } + + // point interpolation does not require the end key time window interpolation. + if (isPointInterpoQuery(pQuery)) { + return; + } + + // interpolation query does not generate the time window end interpolation + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1) * step; + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pBlock->info.window.ekey:pBlock->info.window.skey; + bool interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } } else { - /* - * the sqlfunctionCtx parameters should be set done before all functions are invoked, - * since the selectivity + tag_prj query needs all parameters been set done. - * tag_prj function are changed to be TSDB_FUNC_TAG_DUMMY - */ - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - pCtx[k].nStartQueryTimestamp = pDataBlockInfo->window.skey; - aAggs[functionId].xFunction(&pCtx[k]); + setNotInterpoWindowKey(pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + } +} + +static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t groupId) { + STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; + int32_t numOfOutput = pOperatorInfo->numOfOutput; + SQuery* pQuery = pRuntimeEnv->pQuery; + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + + int32_t prevIndex = curTimeWindowIndex(pResultRowInfo); + + TSKEY* tsCols = NULL; + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); + tsCols = (int64_t*) pColDataInfo->pData; + assert(tsCols[0] == pSDataBlock->info.window.skey && + tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); + } + + int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); + TSKEY ts = getStartTsKey(pQuery, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + + STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQuery); + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + + SResultRow* pResult = NULL; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &win, masterScan, &pResult, groupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t forwardStep = 0; + TSKEY ekey = reviseWindowEkey(pQuery, &win); + forwardStep = + getNumOfRowsInTimeWindow(pQuery, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // prev time window not interpolation yet. + int32_t curIndex = curTimeWindowIndex(pResultRowInfo); + if (prevIndex != -1 && prevIndex < curIndex && pQuery->timeWindowInterpo) { + for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. + SResultRow* pRes = pResultRowInfo->pResult[j]; + if (pRes->closed) { + assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && + resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + continue; + } + + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &w, masterScan, &pResult, groupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + + assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY *)pRuntimeEnv->prevRow[0], + -1, tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &win, masterScan, &pResult, groupId, pInfo->pCtx, + numOfOutput, pInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } } - _end: - if (pRuntimeEnv->timeWindowInterpo) { - int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; - saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, rowIndex); + // window start key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = (forwardStep - 1) * step + startPos; + startPos = getNextQualifiedWindow(pQuery, &nextWin, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pResultRowInfo, &nextWin, masterScan, &pResult, groupId, + pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ekey = reviseWindowEkey(pQuery, &nextWin); + forwardStep = getNumOfRowsInTimeWindow(pQuery, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + + // window start(end) key interpolation + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &nextWin, startPos, forwardStep); + doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + } + + if (pQuery->timeWindowInterpo) { + int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; + saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); } + + updateResultRowInfoActiveIndex(pResultRowInfo, pQuery, pQuery->current->lastKey); } -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; +static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->pQuery->current; - // not assign result buffer yet, add new result buffer, TODO remove it - char* d = pData; - int16_t len = bytes; - if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) { - d = varDataVal(pData); - len = varDataLen(pData); - } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); - return -1; + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); + int16_t bytes = pColInfoData->info.bytes; + int16_t type = pColInfoData->info.type; + SQuery *pQuery = pRuntimeEnv->pQuery; + + if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { + qError("QInfo:%"PRIu64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); + return; } - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, groupIndex); - assert (pResultRow != NULL); + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + char* val = ((char*)pColInfoData->pData) + bytes * j; + if (isNull(val, type)) { + continue; + } + + // Compare with the previous row of this column, and do not set the output buffer again if they are identical. + if (pInfo->prevData == NULL || (memcmp(pInfo->prevData, val, bytes) != 0)) { + if (pInfo->prevData == NULL) { + pInfo->prevData = malloc(bytes); + } + + memcpy(pInfo->prevData, val, bytes); + + if (pQuery->stableQuery && pQuery->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); + } + + int32_t ret = + setGroupResultOutputBuf(pRuntimeEnv, pInfo, pOperator->numOfOutput, val, type, bytes, item->groupIndex); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + } + + for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { + pInfo->binfo.pCtx[k].size = 1; + int32_t functionId = pInfo->binfo.pCtx[k].functionId; + if (functionNeedToExecute(pRuntimeEnv, &pInfo->binfo.pCtx[k], functionId)) { + aAggs[functionId].xFunctionF(&pInfo->binfo.pCtx[k], j); + } + } + } +} + +static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + STableQueryInfo* item = pRuntimeEnv->pQuery->current; + + // primary timestamp column + SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, 0); + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + SOptrBasicInfo* pBInfo = &pInfo->binfo; + + int64_t gap = pOperator->pRuntimeEnv->pQuery->sw.gap; + pInfo->numOfRows = 0; + + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { + if (pInfo->prevTs == INT64_MIN) { + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows = 1; + pInfo->start = j; + } else if (tsList[j] - pInfo->prevTs <= gap) { + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows += 1; + pInfo->start = j; + } else { // start a new session window + SResultRow* pResult = NULL; + + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); + + pInfo->curWindow.skey = tsList[j]; + pInfo->curWindow.ekey = tsList[j]; + pInfo->prevTs = tsList[j]; + pInfo->numOfRows = 1; + pInfo->start = j; + } + } + + SResultRow* pResult = NULL; + + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, &pBInfo->resultRowInfo, &pInfo->curWindow, masterScan, + &pResult, item->groupIndex, pBInfo->pCtx, pOperator->numOfOutput, + pBInfo->rowCellInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + } + doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, + pSDataBlock->info.rows, pOperator->numOfOutput); +} + +static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { int64_t v = -1; GET_TYPED_DATA(v, int64_t, type, pData); if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { @@ -1303,94 +1447,58 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat pResultRow->win.skey = v; pResultRow->win.ekey = v; } +} + +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SGroupbyOperatorInfo *pInfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { + SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + + int32_t *rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + SResultRowInfo *pResultRowInfo = &pInfo->binfo.resultRowInfo; + SQLFunctionCtx *pCtx = pInfo->binfo.pCtx; + + // not assign result buffer yet, add new result buffer, TODO remove it + char* d = pData; + int16_t len = bytes; + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + d = varDataVal(pData); + len = varDataLen(pData); + } + + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, d, len, true, groupIndex); + assert (pResultRow != NULL); + setResultRowKey(pResultRow, pData, type); if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQuery->resultRowSize); if (ret != 0) { return -1; } } - setResultOutputBuf(pRuntimeEnv, pResultRow); - initCtxOutputBuf(pRuntimeEnv); + setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset); + initCtxOutputBuffer(pCtx, numOfCols); return TSDB_CODE_SUCCESS; } -static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, SArray* pDataBlock) { - SSqlGroupbyExpr *pGroupbyExpr = pQuery->pGroupbyExpr; - +static int32_t getGroupbyColumnIndex(SSqlGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock) { for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) { SColIndex* pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, k); if (TSDB_COL_IS_TAG(pColIndex->flag)) { continue; } - int16_t colIndex = -1; int32_t colId = pColIndex->colId; - for (int32_t i = 0; i < pQuery->numOfCols; ++i) { - if (pQuery->colList[i].colId == colId) { - colIndex = i; - break; - } - } - - assert(colIndex >= 0 && colIndex < pQuery->numOfCols); - - *type = pQuery->colList[colIndex].type; - *bytes = pQuery->colList[colIndex].bytes; - /* - * the colIndex is acquired from the first tables of all qualified tables in this vnode during query prepare - * stage, the remain tables may not have the required column in cache actually. So, the validation of required - * column in cache with the corresponding schema is reinforced. - */ - int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *p = taosArrayGet(pDataBlock, i); - if (pColIndex->colId == p->info.colId) { - return p->pData; + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + if (pColInfo->info.colId == colId) { + return i; } } } - return NULL; -} - -static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - - // compare tag first - if (tVariantCompare(&pCtx[0].tag, elem.tag) != 0) { - return TS_JOIN_TAG_NOT_EQUALS; - } - - TSKEY key = *(TSKEY *)((char*)pCtx[0].aInputElemBuf + TSDB_KEYSIZE * offset); - -#if defined(_DEBUG_VIEW) - printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", - elem.ts, key, elem.tag.i64, pQuery->order.order, pRuntimeEnv->pTsBuf->tsOrder, - pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); -#endif - - if (QUERY_IS_ASC_QUERY(pQuery)) { - if (key < elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key > elem.ts) { - assert(false); - } - } else { - if (key > elem.ts) { - return TS_JOIN_TS_NOT_EQUALS; - } else if (key < elem.ts) { - assert(false); - } - } - - return TS_JOIN_TS_EQUAL; + assert(0); + return -1; } static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { @@ -1415,485 +1523,100 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return pCtx->param[0].i64 == pQuery->order.order; } - // in the supplementary scan, only the following functions need to be executed - if (IS_REVERSE_SCAN(pRuntimeEnv)) { + // in the reverse table scan, only the following functions need to be executed + if (IS_REVERSE_SCAN(pRuntimeEnv) || + (pRuntimeEnv->scanFlag == REPEAT_SCAN && functionId != TSDB_FUNC_STDDEV && functionId != TSDB_FUNC_PERCT)) { return false; } return true; } -void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SQuery* pQuery = pRuntimeEnv->pQuery; - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionId != TSDB_FUNC_TWA) { - pRuntimeEnv->pCtx[k].start.key = INT64_MIN; - continue; - } +void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColIndex* pColIndex) { + SDataStatis *pStatis = NULL; - SColIndex* pColIndex = &pQuery->pExpr1[k].base.colInfo; - int16_t index = pColIndex->colIndex; - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); + if (pSDataBlock->pBlockStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { + pStatis = &pSDataBlock->pBlockStatis[pColIndex->colIndex]; - assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); - double v1 = 0, v2 = 0, v = 0; - - if (prevRowIndex == -1) { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]); - } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); - } - - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); - - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = curTs, .val = &v2}; - SPoint point = (SPoint){.key = windowKey, .val = &v}; - taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); - - if (type == RESULT_ROW_START_INTERP) { - pRuntimeEnv->pCtx[k].start.key = point.key; - pRuntimeEnv->pCtx[k].start.val = v; - } else { - pRuntimeEnv->pCtx[k].end.key = point.key; - pRuntimeEnv->pCtx[k].end.val = v; - } - } -} - -static void setTimeWindowSKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; - if (key == ts) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } else if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && prevTs < key) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > key))) { - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - } - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pRuntimeEnv->pCtx[k].size = 1; - } + pCtx->preAggVals.statis = *pStatis; + pCtx->preAggVals.isSet = true; + assert(pCtx->preAggVals.statis.numOfNull <= pSDataBlock->info.rows); } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + pCtx->preAggVals.isSet = false; } -} -static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + pCtx->hasNull = hasNullRv(pColIndex, pStatis); - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - pRuntimeEnv->pCtx[i].size = 0; + // set the statistics data for primary time stamp column + if (pCtx->functionId == TSDB_FUNC_SPREAD && pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + pCtx->preAggVals.isSet = true; + pCtx->preAggVals.statis.min = pSDataBlock->info.window.skey; + pCtx->preAggVals.statis.max = pSDataBlock->info.window.ekey; } } -static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { - SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); - - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* item = pQuery->current; - - int64_t groupId = item->groupIndex; - - SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0); - - TSKEY *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL; - bool groupbyColumnValue = pRuntimeEnv->groupbyColumn; - - int16_t type = 0; - int16_t bytes = 0; - - char *groupbyColumnData = NULL; - if (groupbyColumnValue) { - groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock); - } - - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - char *dataBlock = getDataBlock(pRuntimeEnv, &pRuntimeEnv->sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &pRuntimeEnv->sasArray[k], k, pQInfo->vgId); - pCtx[k].size = 1; - } - - // set the input column data - for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { - SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k]; - pFilterInfo->pData = getDataBlockImpl(pDataBlock, pFilterInfo->info.colId); - assert(pFilterInfo->pData != NULL); +// set the output buffer for the selectivity + tag query +static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { + if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { + return TSDB_CODE_SUCCESS; } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + int32_t num = 0; + int16_t tagLen = 0; - // from top to bottom in desc - // from bottom to top in asc order - if (pRuntimeEnv->pTsBuf != NULL) { - qDebug("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows, - pQuery->order.order, pRuntimeEnv->pTsBuf->cur.order); + SQLFunctionCtx* p = NULL; + SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; } - int32_t offset = -1; - TSKEY prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0]; - int32_t prevRowIndex = -1; - - for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { - offset = GET_COL_DATA_POS(pQuery, j, step); - - if (pRuntimeEnv->pTsBuf != NULL) { - int32_t ret = doTSJoinFilter(pRuntimeEnv, offset); - if (ret == TS_JOIN_TAG_NOT_EQUALS) { - break; - } else if (ret == TS_JOIN_TS_NOT_EQUALS) { - continue; - } else { - assert(ret == TS_JOIN_TS_EQUAL); - } - } - - if (pQuery->numOfFilterCols > 0 && (!doFilterData(pQuery, offset))) { + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functionId = pCtx[i].functionId; + + if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { + tagLen += pCtx[i].outputBytes; + pTagCtx[num++] = &pCtx[i]; + } else if ((aAggs[functionId].status & TSDB_FUNCSTATE_SELECTIVITY) != 0) { + p = &pCtx[i]; + } else if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG) { + // tag function may be the group by tag column + // ts may be the required primary timestamp column continue; - } - - // interval window query, decide the time window according to the primary timestamp - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); - int64_t ts = tsCols[offset]; - - STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - - SResultRow* pResult = NULL; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { // null data, too many state code - goto _end; - } - - // window start key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - // check for the time window end time interpolation - int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { - for (int32_t k = prevWindowIndex; k < curIndex; ++k) { - SResultRow *pRes = pWindowResInfo->pResult[k]; - if (pRes->closed) { - assert(resultRowInterpolated(pResult, RESULT_ROW_START_INTERP) && resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - continue; - } - - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &pRes->win, masterScan, &pResult, groupId); - assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - - setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win); - doRowwiseApplyFunctions(pRuntimeEnv, &pRes->win, offset); - } - - // restore current time window - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - continue; - } - } - - setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win); - } - - doRowwiseApplyFunctions(pRuntimeEnv, &win, offset); - int32_t index = pWindowResInfo->curIndex; - - STimeWindow nextWin = win; - while (1) { - getNextTimeWindow(pQuery, &nextWin); - if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (nextWin.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { - break; - } - - if (ts < nextWin.skey || ts > nextWin.ekey) { - break; - } - - // null data, failed to allocate more memory buffer - int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - break; - } - - setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); - doRowwiseApplyFunctions(pRuntimeEnv, &nextWin, offset); - } - - // restore the index, add the result row will move the index - pWindowResInfo->curIndex = index; - } else { // other queries - // decide which group this rows belongs to according to current state value - if (groupbyColumnValue) { - char *val = groupbyColumnData + bytes * offset; - if (isNull(val, type)) { // ignore the null value - continue; - } - - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - } - - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - int32_t functionId = pQuery->pExpr1[k].base.functionId; - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { - aAggs[functionId].xFunctionF(&pCtx[k], offset); - } - } - } - - prevTs = tsCols[offset]; - prevRowIndex = offset; - - if (pRuntimeEnv->pTsBuf != NULL) { - // if timestamp filter list is empty, quit current query - if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { - setQueryStatus(pQuery, QUERY_COMPLETED); - break; - } - } - } - - _end: - assert(offset >= 0 && tsCols != NULL); - if (prevTs != INT64_MIN && prevTs != *(int64_t*)pRuntimeEnv->prevRow[0]) { - assert(prevRowIndex >= 0); - item->lastKey = prevTs + step; - } - - // In case of all rows in current block are not qualified - if (pRuntimeEnv->timeWindowInterpo && prevRowIndex != -1) { - saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock, prevRowIndex); - } - - if (pRuntimeEnv->pTsBuf != NULL) { - item->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - } -} - -static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, - SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - STableQueryInfo* pTableQueryInfo = pQuery->current; - SResultRowInfo* pResultRowInfo = &pRuntimeEnv->windowResInfo; - - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); - } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); - } - - // update the lastkey of current table for projection/aggregation query - TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey; - pTableQueryInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - // interval query with limit applied - int32_t numOfRes = 0; - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { - numOfRes = pResultRowInfo->size; - updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); - } else { // projection query - numOfRes = (int32_t) getNumOfResult(pRuntimeEnv); - - // update the number of output result - if (numOfRes > 0 && pQuery->checkResultBuf == 1) { - assert(numOfRes >= pQuery->rec.rows); - pQuery->rec.rows = numOfRes; - - if (numOfRes >= pQuery->rec.threshold) { - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - } - - if ((pQuery->limit.limit >= 0) && (pQuery->limit.limit + pQuery->limit.offset) <= numOfRes) { - setQueryStatus(pQuery, QUERY_COMPLETED); - } - - if (((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || - ((pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { - setQueryStatus(pQuery, QUERY_COMPLETED); - } + } else { + // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ } } - - return numOfRes; -} - -void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId) { - - int32_t functionId = pQuery->pExpr1[colIndex].base.functionId; - int32_t colId = pQuery->pExpr1[colIndex].base.colInfo.colId; - - SDataStatis *tpField = NULL; - pCtx->hasNull = hasNullValue(&pQuery->pExpr1[colIndex].base.colInfo, pStatis, &tpField); - pCtx->aInputElemBuf = inputData; - - if (tpField != NULL) { - pCtx->preAggVals.isSet = true; - pCtx->preAggVals.statis = *tpField; - assert(pCtx->preAggVals.statis.numOfNull <= pBlockInfo->rows); + if (p != NULL) { + p->tagInfo.pTagCtxList = pTagCtx; + p->tagInfo.numOfTagCols = num; + p->tagInfo.tagsLen = tagLen; } else { - pCtx->preAggVals.isSet = false; - } - - pCtx->preAggVals.dataBlockLoaded = (inputData != NULL); - - // limit/offset query will affect this value - pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1; - - // minimum value no matter ascending/descending order query - pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size + 1); - assert(pCtx->startOffset >= 0); - - uint32_t status = aAggs[functionId].nStatus; - if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) { - pCtx->ptsList = tsCol; - } - - if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) { - // last_dist or first_dist function - // store the first&last timestamp into the intermediate buffer [1], the true - // value may be null but timestamp will never be null - } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA || - functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { - /* - * least squares function needs two columns of input, currently, the x value of linear equation is set to - * timestamp column, and the y-value is the column specified in pQuery->pExpr1[i].colIdxInBuffer - * - * top/bottom function needs timestamp to indicate when the - * top/bottom values emerge, so does diff function - */ - if (functionId == TSDB_FUNC_TWA) { - pCtx->param[1].i64 = pQuery->window.skey; - pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[2].i64 = pQuery->window.ekey; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - } - - } else if (functionId == TSDB_FUNC_ARITHM) { - pCtx->param[1].pz = param; - } else if (functionId == TSDB_FUNC_SPREAD) { // set the statistics data for primary time stamp column - if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - pCtx->preAggVals.isSet = true; - pCtx->preAggVals.statis.min = pBlockInfo->window.skey; - pCtx->preAggVals.statis.max = pBlockInfo->window.ekey; - } - } else if (functionId == TSDB_FUNC_INTERP) { - SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); - - SInterpInfoDetail *pInterpInfo = (SInterpInfoDetail *)GET_ROWCELL_INTERBUF(pInfo); - pInterpInfo->type = (int8_t)pQuery->fillType; - pInterpInfo->ts = pQuery->window.skey; - pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - - if (pQuery->fillVal != NULL) { - if (isNull((const char*) &pQuery->fillVal[colIndex], pCtx->inputType)) { - pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; - } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value - if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[1], (char*) &pQuery->fillVal[colIndex], pCtx->inputBytes, pCtx->inputType); - } - } - } - } else if (functionId == TSDB_FUNC_TS_COMP) { - pCtx->param[0].i64 = vgId; - pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; + tfree(pTagCtx); } -#if defined(_DEBUG_VIEW) - // int64_t *tsList = (int64_t *)primaryColumnData; -// int64_t s = tsList[0]; -// int64_t e = tsList[size - 1]; - -// if (IS_DATA_BLOCK_LOADED(blockStatus)) { -// qDebug("QInfo:%p query ts:%lld-%lld, offset:%d, rows:%d, bstatus:%d, -// functId:%d", GET_QINFO_ADDR(pQuery), -// s, e, startOffset, size, blockStatus, functionId); -// } else { -// qDebug("QInfo:%p block not loaded, bstatus:%d", -// GET_QINFO_ADDR(pQuery), blockStatus); -// } -#endif + return TSDB_CODE_SUCCESS; } -// set the output buffer for the selectivity + tag query -static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, + int32_t** rowCellInfoOffset) { SQuery* pQuery = pRuntimeEnv->pQuery; - if (isSelectivityWithTagsQuery(pQuery)) { - int32_t num = 0; - int16_t tagLen = 0; - - SQLFunctionCtx *p = NULL; - SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); - if (pTagCtx == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg *pSqlFuncMsg = &pQuery->pExpr1[i].base; - - if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { - tagLen += pCtx[i].outputBytes; - pTagCtx[num++] = &pCtx[i]; - } else if ((aAggs[pSqlFuncMsg->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) { - p = &pCtx[i]; - } else if (pSqlFuncMsg->functionId == TSDB_FUNC_TS || pSqlFuncMsg->functionId == TSDB_FUNC_TAG) { - // tag function may be the group by tag column - // ts may be the required primary timestamp column - continue; - } else { - // the column may be the normal column, group by normal_column, the functionId is TSDB_FUNC_PRJ - } - } - if (p != NULL) { - p->tagInfo.pTagCtxList = pTagCtx; - p->tagInfo.numOfTagCols = num; - p->tagInfo.tagsLen = tagLen; - } else { - tfree(pTagCtx); - } + SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + if (pFuncCtx == NULL) { + return NULL; } - return TSDB_CODE_SUCCESS; -} - -static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { - qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); - SQuery *pQuery = pRuntimeEnv->pQuery; - - pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); - pRuntimeEnv->offset = calloc(pQuery->numOfOutput, sizeof(int16_t)); - pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); - pRuntimeEnv->sasArray = calloc(pQuery->numOfOutput, sizeof(SArithmeticSupport)); - - if (pRuntimeEnv->offset == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL || pRuntimeEnv->sasArray == NULL) { - goto _clean; + *rowCellInfoOffset = calloc(numOfOutput, sizeof(int32_t)); + if (*rowCellInfoOffset == 0) { + tfree(pFuncCtx); + return NULL; } - pRuntimeEnv->offset[0] = 0; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg *pSqlFuncMsg = &pQuery->pExpr1[i].base; + for (int32_t i = 0; i < numOfOutput; ++i) { + SSqlFuncMsg *pSqlFuncMsg = &pExpr[i].base; + SQLFunctionCtx* pCtx = &pFuncCtx[i]; - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - SColIndex* pIndex = &pSqlFuncMsg->colInfo; + SColIndex *pIndex = &pSqlFuncMsg->colInfo; if (TSDB_COL_REQ_NULL(pIndex->flag)) { pCtx->requireNull = true; @@ -1902,35 +1625,18 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->requireNull = false; } - int32_t index = pSqlFuncMsg->colInfo.colIndex; - if (TSDB_COL_IS_TAG(pIndex->flag)) { - if (pIndex->colId == TSDB_TBNAME_COLUMN_INDEX) { // todo refactor - SSchema s = tGetTableNameColumnSchema(); - - pCtx->inputBytes = s.bytes; - pCtx->inputType = s.type; - } else { - pCtx->inputBytes = pQuery->tagColList[index].bytes; - pCtx->inputType = pQuery->tagColList[index].type; - } - } else if (TSDB_COL_IS_UD_COL(pIndex->flag)) { - pCtx->inputBytes = pSqlFuncMsg->arg[0].argBytes; - pCtx->inputType = pSqlFuncMsg->arg[0].argType; - } else { - pCtx->inputBytes = pQuery->colList[index].bytes; - pCtx->inputType = pQuery->colList[index].type; - } + pCtx->inputBytes = pSqlFuncMsg->colBytes; + pCtx->inputType = pSqlFuncMsg->colType; - assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; - pCtx->outputBytes = pQuery->pExpr1[i].bytes; - pCtx->outputType = pQuery->pExpr1[i].type; + pCtx->outputBytes = pExpr[i].bytes; + pCtx->outputType = pExpr[i].type; pCtx->order = pQuery->order.order; pCtx->functionId = pSqlFuncMsg->functionId; - pCtx->stableQuery = pRuntimeEnv->stableQuery; - pCtx->interBufBytes = pQuery->pExpr1[i].interBytes; + pCtx->stableQuery = pQuery->stableQuery; + pCtx->interBufBytes = pExpr[i].interBytes; pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; @@ -1938,6 +1644,10 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order for (int32_t j = 0; j < pCtx->numOfParams; ++j) { int16_t type = pSqlFuncMsg->arg[j].argType; int16_t bytes = pSqlFuncMsg->arg[j].argBytes; + if (pSqlFuncMsg->functionId == TSDB_FUNC_STDDEV_DST) { + continue; + } + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); } else { @@ -1949,137 +1659,238 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int32_t functionId = pCtx->functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - int32_t f = pQuery->pExpr1[0].base.functionId; + int32_t f = pExpr[0].base.functionId; assert(f == TSDB_FUNC_TS || f == TSDB_FUNC_TS_DUMMY); - pCtx->param[2].i64 = order; + pCtx->param[2].i64 = pQuery->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[3].i64 = functionId; pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[1].i64 = pQuery->order.orderColId; - } - - if (functionId == TSDB_FUNC_ARITHM) { - pRuntimeEnv->sasArray[i].data = calloc(pQuery->numOfCols, POINTER_BYTES); - if (pRuntimeEnv->sasArray[i].data == NULL) { - goto _clean; + } else if (functionId == TSDB_FUNC_INTERP) { + pCtx->param[2].i64 = (int8_t)pQuery->fillType; + if (pQuery->fillVal != NULL) { + if (isNull((const char *)&pQuery->fillVal[i], pCtx->inputType)) { + pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + } else { // todo refactor, tVariantCreateFromBinary should handle the NULL value + if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { + tVariantCreateFromBinary(&pCtx->param[1], (char *)&pQuery->fillVal[i], pCtx->inputBytes, pCtx->inputType); + } + } } + } else if (functionId == TSDB_FUNC_TS_COMP) { + pCtx->param[0].i64 = pQuery->vgId; //TODO this should be the parameter from client + pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == TSDB_FUNC_TWA) { + pCtx->param[1].i64 = pQuery->window.skey; + pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[2].i64 = pQuery->window.ekey; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + } else if (functionId == TSDB_FUNC_ARITHM) { + pCtx->param[1].pz = (char*) &pRuntimeEnv->sasArray[i]; } + } - if (i > 0) { - pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; - pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pExpr1[i - 1].interBytes; - } + for(int32_t i = 1; i < numOfOutput; ++i) { + (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowCellInfo) + + pExpr[i - 1].interBytes * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery)); } - *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; + setCtxTagColumnInfo(pFuncCtx, numOfOutput); - // if it is group by normal column, do not set output buffer, the output buffer is pResult - // fixed output query/multi-output query for normal table - if (!pRuntimeEnv->groupbyColumn && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { - resetDefaultResInfoOutputBuf(pRuntimeEnv); - } + return pFuncCtx; +} - if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { - goto _clean; +static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + if (pCtx == NULL) { + return NULL; } - qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv)); - return TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { + tVariantDestroy(&pCtx[i].param[j]); + } -_clean: - tfree(pRuntimeEnv->pCtx); - tfree(pRuntimeEnv->offset); - tfree(pRuntimeEnv->rowCellInfoOffset); - tfree(pRuntimeEnv->sasArray); + tVariantDestroy(&pCtx[i].tag); + tfree(pCtx[i].tagInfo.pTagCtxList); + } - return TSDB_CODE_QRY_OUT_OF_MEMORY; + tfree(pCtx); + return NULL; } -static void doFreeQueryHandle(SQInfo* pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; +static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfTables) { + qDebug("QInfo:%"PRIu64" setup runtime env", GET_QID(pRuntimeEnv)); + SQuery *pQuery = pRuntimeEnv->pQuery; - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); + pRuntimeEnv->prevGroupId = INT32_MIN; + pRuntimeEnv->pQuery = pQuery; - pRuntimeEnv->pQueryHandle = NULL; - pRuntimeEnv->pSecQueryHandle = NULL; + pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pRuntimeEnv->keyBuf = malloc(pQuery->maxSrcColumnSize + sizeof(int64_t)); + pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); + pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQuery->numOfCols + pQuery->srcRowSize); + pRuntimeEnv->tagVal = malloc(pQuery->tagLen); + pRuntimeEnv->currentOffset = pQuery->limit.offset; - SMemRef* pMemRef = &pQInfo->memRef; - assert(pMemRef->ref == 0 && pMemRef->imem == NULL && pMemRef->mem == NULL); -} + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + pRuntimeEnv->pTableRetrieveTsMap = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + pRuntimeEnv->sasArray = calloc(pQuery->numOfOutput, sizeof(SArithmeticSupport)); -static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { - if (pRuntimeEnv->pQuery == NULL) { - return; + if (pRuntimeEnv->sasArray == NULL || pRuntimeEnv->pResultRowHashTable == NULL || pRuntimeEnv->keyBuf == NULL || + pRuntimeEnv->prevRow == NULL || pRuntimeEnv->tagVal == NULL) { + goto _clean; } - SQuery *pQuery = pRuntimeEnv->pQuery; - SQInfo* pQInfo = (SQInfo*) GET_QINFO_ADDR(pRuntimeEnv); - - qDebug("QInfo:%p teardown runtime env", pQInfo); - cleanupResultRowInfo(&pRuntimeEnv->windowResInfo); - - if (isTSCompQuery(pQuery)) { - FILE *f = *(FILE **)pQuery->sdata[0]->data; - - if (f) { - fclose(f); - *(FILE **)pQuery->sdata[0]->data = NULL; + if (pQuery->numOfCols) { + char* start = POINTER_BYTES * pQuery->numOfCols + (char*) pRuntimeEnv->prevRow; + pRuntimeEnv->prevRow[0] = start; + for(int32_t i = 1; i < pQuery->numOfCols; ++i) { + pRuntimeEnv->prevRow[i] = pRuntimeEnv->prevRow[i - 1] + pQuery->colList[i-1].bytes; } + + *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; } + qDebug("QInfo:%"PRIu64" init runtime environment completed", GET_QID(pRuntimeEnv)); - if (pRuntimeEnv->pCtx != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + // group by normal column, sliding window query, interval query are handled by interval query processor + // interval (down sampling operation) + if (onlyQueryTags(pQuery)) { // do nothing for tags query + + } else if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + if (pQuery->stableQuery) { + pRuntimeEnv->proot = createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, + pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); + } else { + pRuntimeEnv->proot = + createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); - for (int32_t j = 0; j < pCtx->numOfParams; ++j) { - tVariantDestroy(&pCtx->param[j]); + if (pQuery->pExpr2 != NULL) { + pRuntimeEnv->proot = + createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); } - tVariantDestroy(&pCtx->tag); - tfree(pCtx->tagInfo.pTagCtxList); + if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { + SOperatorInfo* pInfo = pRuntimeEnv->proot; + pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput); + } } - tfree(pRuntimeEnv->pCtx); - } + } else if (pQuery->groupbyColumn) { + pRuntimeEnv->proot = + createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); - if (pRuntimeEnv->sasArray != NULL) { - for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { - tfree(pRuntimeEnv->sasArray[i].data); + if (pQuery->pExpr2 != NULL) { + pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); } + } else if (pQuery->sw.gap > 0) { + pRuntimeEnv->proot = createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); - tfree(pRuntimeEnv->sasArray); - } - - pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); + if (pQuery->pExpr2 != NULL) { + pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); + } + } else if (isFixedOutputQuery(pQuery)) { + if (pQuery->stableQuery && !isTsCompQuery(pQuery)) { + pRuntimeEnv->proot = + createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + } else { + pRuntimeEnv->proot = + createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + } - destroyResultBuf(pRuntimeEnv->pResultBuf); - doFreeQueryHandle(pQInfo); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); + + if (pQuery->pExpr2 != NULL) { + pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQuery->pExpr2, pQuery->numOfExpr2); + } + } else { // diff/add/multiply/subtract/division + assert(pQuery->checkResultBuf == 1); + if (!onlyQueryTags(pQuery)) { + pRuntimeEnv->proot = + createArithOperatorInfo(pRuntimeEnv, pRuntimeEnv->pTableScanner, pQuery->pExpr1, pQuery->numOfOutput); + setTableScanFilterOperatorInfo(pRuntimeEnv->pTableScanner->info, pRuntimeEnv->proot); + } + } + + if (pQuery->limit.offset > 0) { + pRuntimeEnv->proot = createOffsetOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); + } + + if (pQuery->limit.limit > 0) { + pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); + } + + return TSDB_CODE_SUCCESS; + +_clean: + tfree(pRuntimeEnv->sasArray); + tfree(pRuntimeEnv->pResultRowHashTable); + tfree(pRuntimeEnv->keyBuf); + tfree(pRuntimeEnv->prevRow); + tfree(pRuntimeEnv->tagVal); + + return TSDB_CODE_QRY_OUT_OF_MEMORY; +} + +static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); + pRuntimeEnv->pQueryHandle = NULL; + + SMemRef* pMemRef = &pQuery->memRef; + assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); +} + +static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { + SQuery *pQuery = pRuntimeEnv->pQuery; + SQInfo* pQInfo = (SQInfo*) pRuntimeEnv->qinfo; + + qDebug("QInfo:%"PRIu64" teardown runtime env", pQInfo->qId); + + if (pRuntimeEnv->sasArray != NULL) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + tfree(pRuntimeEnv->sasArray[i].data); + tfree(pRuntimeEnv->sasArray[i].colList); + } + + tfree(pRuntimeEnv->sasArray); + } + + destroyResultBuf(pRuntimeEnv->pResultBuf); + doFreeQueryHandle(pRuntimeEnv); pRuntimeEnv->pTsBuf = tsBufDestroy(pRuntimeEnv->pTsBuf); - tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->keyBuf); - tfree(pRuntimeEnv->rowCellInfoOffset); tfree(pRuntimeEnv->prevRow); + tfree(pRuntimeEnv->tagVal); taosHashCleanup(pRuntimeEnv->pResultRowHashTable); pRuntimeEnv->pResultRowHashTable = NULL; pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); + taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); + pRuntimeEnv->prevResult = NULL; + + taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); + pRuntimeEnv->pTableRetrieveTsMap = NULL; + + destroyOperatorInfo(pRuntimeEnv->proot); } static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { return pQInfo->rspContext != NULL; } -#define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) - -static bool isQueryKilled(SQInfo *pQInfo) { +bool isQueryKilled(SQInfo *pQInfo) { if (IS_QUERY_KILLED(pQInfo)) { return true; } @@ -2090,24 +1901,23 @@ static bool isQueryKilled(SQInfo *pQInfo) { (!needBuildResAfterQueryComplete(pQInfo))) { assert(pQInfo->startExecTs != 0); - qDebug("QInfo:%p retrieve not arrive beyond %d sec, abort current query execution, start:%"PRId64", current:%d", pQInfo, 1, - pQInfo->startExecTs, taosGetTimestampSec()); + qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d sec, abort current query execution, start:%" PRId64 + ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); return true; } return false; } -static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} +void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} -static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { - SQuery* pQuery = pRuntimeEnv->pQuery; +static bool isFixedOutputQuery(SQuery* pQuery) { if (QUERY_IS_INTERVAL_QUERY(pQuery)) { return false; } // Note:top/bottom query is fixed output query - if (pRuntimeEnv->topBotQuery || pRuntimeEnv->groupbyColumn) { + if (pQuery->topBotQuery || pQuery->groupbyColumn || isTsCompQuery(pQuery)) { return true; } @@ -2124,7 +1934,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { continue; } - if (!IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus)) { + if (!IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].status)) { return true; } } @@ -2145,7 +1955,7 @@ bool isPointInterpoQuery(SQuery *pQuery) { } // TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION -static bool isSumAvgRateQuery(SQuery *pQuery) { +static UNUSED_FUNC bool isSumAvgRateQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pExpr1[i].base.functionId; if (functionId == TSDB_FUNC_TS) { @@ -2199,7 +2009,7 @@ static bool needReverseScan(SQuery *pQuery) { * The following 4 kinds of query are treated as the tags query * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query */ -static bool onlyQueryTags(SQuery* pQuery) { +bool onlyQueryTags(SQuery* pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQuery->pExpr1[i]; @@ -2239,7 +2049,7 @@ void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int6 static void setScanLimitationByResultBuffer(SQuery *pQuery) { if (isTopBottomQuery(pQuery)) { pQuery->checkResultBuf = 0; - } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { + } else if (isGroupbyColumn(pQuery->pGroupbyExpr)) { pQuery->checkResultBuf = 0; } else { bool hasMultioutput = false; @@ -2249,7 +2059,7 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { continue; } - hasMultioutput = IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].nStatus); + hasMultioutput = IS_MULTIOUTPUT(aAggs[pExprMsg->functionId].status); if (!hasMultioutput) { break; } @@ -2262,11 +2072,11 @@ static void setScanLimitationByResultBuffer(SQuery *pQuery) { /* * todo add more parameters to check soon.. */ -bool colIdCheck(SQuery *pQuery) { +bool colIdCheck(SQuery *pQuery, uint64_t qId) { // load data column information is incorrect for (int32_t i = 0; i < pQuery->numOfCols - 1; ++i) { if (pQuery->colList[i].colId == pQuery->colList[i + 1].colId) { - qError("QInfo:%p invalid data load column for query", GET_QINFO_ADDR(pQuery)); + qError("QInfo:%"PRIu64" invalid data load column for query", qId); return false; } } @@ -2297,11 +2107,11 @@ static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSD static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); } -// todo refactor, add iterator static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) { - size_t t = taosArrayGetSize(pQInfo->tableGroupInfo.pGroupList); + SQuery* pQuery = &pQInfo->query; + size_t t = taosArrayGetSize(pQuery->tableGroupInfo.pGroupList); for(int32_t i = 0; i < t; ++i) { - SArray* p1 = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + SArray* p1 = taosArrayGetP(pQuery->tableGroupInfo.pGroupList, i); size_t len = taosArrayGetSize(p1); for(int32_t j = 0; j < len; ++j) { @@ -2319,13 +2129,13 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo SQuery* pQuery = pQInfo->runtimeEnv.pQuery; // in case of point-interpolation query, use asc order scan - char msg[] = "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 + char msg[] = "QInfo:%"PRIu64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" PRId64 "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; // todo handle the case the the order irrelevant query type mixed up with order critical query type // descending order query for last_row query if (isFirstLastRowQuery(pQuery)) { - qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", pQInfo, pQuery->order.order, TSDB_ORDER_ASC); + qDebug("QInfo:%"PRIu64" scan order changed for last_row query, old:%d, new:%d", pQInfo->qId, pQuery->order.order, TSDB_ORDER_ASC); pQuery->order.order = TSDB_ORDER_ASC; if (pQuery->window.skey > pQuery->window.ekey) { @@ -2335,7 +2145,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo return; } - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) { + if (isGroupbyColumn(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) { pQuery->order.order = TSDB_ORDER_ASC; if (pQuery->window.skey > pQuery->window.ekey) { SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2347,8 +2157,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo if (isPointInterpoQuery(pQuery) && pQuery->interval.interval == 0) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, - pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + qDebug(msg, pQInfo->qId, "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); } @@ -2359,7 +2168,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo if (pQuery->interval.interval == 0) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, + qDebug(msg, pQInfo->qId, "only-first", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2369,7 +2178,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo pQuery->order.order = TSDB_ORDER_ASC; } else if (onlyLastQuery(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "only-last", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, + qDebug(msg, pQInfo->qId, "only-last", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2383,7 +2192,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo if (stableQuery) { if (onlyFirstQuery(pQuery)) { if (!QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "only-first stable", pQuery->order.order, TSDB_ORDER_ASC, + qDebug(msg, pQInfo->qId, "only-first stable", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2393,7 +2202,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo pQuery->order.order = TSDB_ORDER_ASC; } else if (onlyLastQuery(pQuery)) { if (QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "only-last stable", pQuery->order.order, TSDB_ORDER_DESC, + qDebug(msg, pQInfo->qId, "only-last stable", pQuery->order.order, TSDB_ORDER_DESC, pQuery->window.skey, pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); @@ -2406,48 +2215,29 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo } } -static int32_t getInitialPageNum(SQInfo *pQInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - int32_t INITIAL_RESULT_ROWS_VALUE = 16; - - int32_t num = 0; - - if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { - num = 128; - } else if (QUERY_IS_INTERVAL_QUERY(pQuery)) { // time window query, allocate one page for each table - size_t s = pQInfo->tableqinfoGroupInfo.numOfTables; - num = (int32_t)(MAX(s, INITIAL_RESULT_ROWS_VALUE)); - } else { // for super table query, one page for each subset - num = 1; // pQInfo->pSidSet->numOfSubSet; - } - - assert(num > 0); - return num; -} - static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { SQuery* pQuery = pRuntimeEnv->pQuery; int32_t MIN_ROWS_PER_PAGE = 4; - *rowsize = (int32_t)(pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery)); + *rowsize = (int32_t)(pQuery->resultRowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery)); int32_t overhead = sizeof(tFilePage); // one page contains at least two rows *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { - *ps = (*ps << 1u); + *ps = ((*ps) << 1u); } - pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); - assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); +// pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); +// assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) -static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, - int32_t numOfRows) { +static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { SQuery* pQuery = pRuntimeEnv->pQuery; - if (pDataStatis == NULL || (pQuery->numOfFilterCols == 0 && (!pRuntimeEnv->topBotQuery))) { + + if (pDataStatis == NULL || pQuery->numOfFilterCols == 0) { return true; } @@ -2506,15 +2296,6 @@ static bool needToLoadDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis *pDat } } - if (pRuntimeEnv->topBotQuery) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - return topbot_datablock_filter(&pCtx[i], functionId, (char *)&pDataStatis[i].min, (char *)&pDataStatis[i].max); - } - } - } - return false; } @@ -2567,87 +2348,325 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { return false; } -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { - *status = BLK_DATA_NO_NEEDED; +static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key, bool ascQuery) { + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - SQuery *pQuery = pRuntimeEnv->pQuery; - int64_t groupId = pQuery->current->groupIndex; +#if defined(_DEBUG_VIEW) + printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n", + elem.ts, key, elem.tag.i64, pQuery->order.order, pRuntimeEnv->pTsBuf->tsOrder, + pRuntimeEnv->pTsBuf->cur.order, pRuntimeEnv->pTsBuf->cur.tsIndex); +#endif + + if (ascQuery) { + if (key < elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key > elem.ts) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); + } + } else { + if (key > elem.ts) { + return TS_JOIN_TS_NOT_EQUALS; + } else if (key < elem.ts) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_INCONSISTAN); + } + } - SQueryCostInfo* pCost = &pRuntimeEnv->summary; + return TS_JOIN_TS_EQUAL; +} + +void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, + SSDataBlock* pBlock, bool ascQuery) { + int32_t numOfRows = pBlock->info.rows; + + int8_t *p = calloc(numOfRows, sizeof(int8_t)); + bool all = true; - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf > 0) { - *status = BLK_DATA_ALL_NEEDED; - } else { // check if this data block is required to load + if (pRuntimeEnv->pTsBuf != NULL) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); + + TSKEY* k = (TSKEY*) pColInfoData->pData; + for (int32_t i = 0; i < numOfRows; ++i) { + int32_t offset = ascQuery? i:(numOfRows - i - 1); + int32_t ret = doTSJoinFilter(pRuntimeEnv, k[offset], ascQuery); + if (ret == TS_JOIN_TAG_NOT_EQUALS) { + break; + } else if (ret == TS_JOIN_TS_NOT_EQUALS) { + all = false; + continue; + } else { + assert(ret == TS_JOIN_TS_EQUAL); + p[offset] = true; + } - // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, do not load current data block. - if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) { - *status = BLK_DATA_ALL_NEEDED; + if (!tsBufNextPos(pRuntimeEnv->pTsBuf)) { + break; + } } - if ((*status) != BLK_DATA_ALL_NEEDED) { - // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, - // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - SResultRow* pResult = NULL; + // save the cursor status + pRuntimeEnv->pQuery->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + } else { + for (int32_t i = 0; i < numOfRows; ++i) { + bool qualified = false; + + for (int32_t k = 0; k < numOfFilterCols; ++k) { + char* pElem = (char*)pFilterInfo[k].pData + pFilterInfo[k].info.bytes * i; + + qualified = false; + for (int32_t j = 0; j < pFilterInfo[k].numOfFilters; ++j) { + SColumnFilterElem* pFilterElem = &pFilterInfo[k].pFilters[j]; + + bool isnull = isNull(pElem, pFilterInfo[k].info.type); + if (isnull) { + if (pFilterElem->fp == isNullOperator) { + qualified = true; + break; + } else { + continue; + } + } else { + if (pFilterElem->fp == notNullOperator) { + qualified = true; + break; + } else if (pFilterElem->fp == isNullOperator) { + continue; + } + } - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + if (pFilterElem->fp(pFilterElem, pElem, pElem, pFilterInfo[k].info.type)) { + qualified = true; + break; + } + } - TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; - STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId) != TSDB_CODE_SUCCESS) { - // todo handle error in set result for timewindow + if (!qualified) { + break; } } - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SSqlFuncMsg* pSqlFunc = &pQuery->pExpr1[i].base; + p[i] = qualified ? 1 : 0; + if (!qualified) { + all = false; + } + } + } - int32_t functionId = pSqlFunc->functionId; - int32_t colId = pSqlFunc->colInfo.colId; - (*status) |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId); - if (((*status) & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { - break; + if (!all) { + int32_t start = 0; + int32_t len = 0; + for (int32_t j = 0; j < numOfRows; ++j) { + if (p[j] == 1) { + len++; + } else { + if (len > 0) { + int32_t cstart = j - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData *pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(((char*)pColumnInfoData->pData) + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); + } + + start += len; + len = 0; } } } + + if (len > 0) { + int32_t cstart = numOfRows - len; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData *pColumnInfoData = taosArrayGet(pBlock->pDataBlock, i); + + int16_t bytes = pColumnInfoData->info.bytes; + memmove(pColumnInfoData->pData + start * bytes, pColumnInfoData->pData + cstart * bytes, len * bytes); + } + + start += len; + len = 0; + } + + pBlock->info.rows = start; + pBlock->pBlockStatis = NULL; // clean the block statistics info + + if (start > 0) { + SColumnInfoData* pColumnInfoData = taosArrayGet(pBlock->pDataBlock, 0); + assert(pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && + pColumnInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); + + pBlock->info.window.skey = *(int64_t*)pColumnInfoData->pData; + pBlock->info.window.ekey = *(int64_t*)(pColumnInfoData->pData + TSDB_KEYSIZE * (start - 1)); + } + } + + tfree(p); +} + +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); +static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes); + +static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { + SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; + uint32_t status = BLK_DATA_NO_NEEDED; + + int32_t numOfOutput = pTableScanInfo->numOfOutput; + for (int32_t i = 0; i < numOfOutput; ++i) { + int32_t functionId = pCtx[i].functionId; + int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId; + + // group by + first/last should not apply the first/last block filter + status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId); + if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) { + return status; + } + } + + return status; +} + +static void doSetFilterColumnInfo(SQuery* pQuery, SSDataBlock* pBlock) { + if (pQuery->numOfFilterCols > 0 && pQuery->pFilterInfo[0].pData != NULL) { + return; + } + + // set the initial static data value filter expression + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j); + + if (pQuery->pFilterInfo[i].info.colId == pColInfo->info.colId) { + pQuery->pFilterInfo[i].pData = pColInfo->pData; + break; + } + } + } +} + +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, + uint32_t* status) { + *status = BLK_DATA_NO_NEEDED; + pBlock->pDataBlock = NULL; + pBlock->pBlockStatis = NULL; + + SQuery* pQuery = pRuntimeEnv->pQuery; + int64_t groupId = pQuery->current->groupIndex; + bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); + + SQInfo* pQInfo = pRuntimeEnv->qinfo; + SQueryCostInfo* pCost = &pQInfo->summary; + + if (pRuntimeEnv->pTsBuf != NULL) { + (*status) = BLK_DATA_ALL_NEEDED; + + if (pQuery->stableQuery) { // todo refactor + SExprInfo* pExprInfo = &pTableScanInfo->pExpr[0]; + int16_t tagId = (int16_t)pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagId); + + // compare tag first + tVariant t = {0}; + doSetTagValueInParam(pQuery->current->pTable, tagId, &t, pColInfo->type, pColInfo->bytes); + setTimestampListJoinInfo(pRuntimeEnv, &t, pQuery->current); + + STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); + if (!tsBufIsValidElem(&elem) || (tsBufIsValidElem(&elem) && (tVariantCompare(&t, elem.tag) != 0))) { + (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; + } + } } + // Calculate all time windows that are overlapping or contain current data block. + // If current data block is contained by all possible time window, do not load current data block. + if (pQuery->numOfFilterCols > 0 || pQuery->groupbyColumn || pQuery->sw.gap > 0 || + (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, &pBlock->info))) { + (*status) = BLK_DATA_ALL_NEEDED; + } + + // check if this data block is required to load + if ((*status) != BLK_DATA_ALL_NEEDED) { + // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet, + // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + SResultRow* pResult = NULL; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; + + STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQuery); + if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } else if (pQuery->stableQuery && (!isTsCompQuery(pQuery))) { // stable aggregate, not interval aggregate or normal column aggregate + doSetTableGroupOutputBuf(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, + pTableScanInfo->rowCellInfoOffset, pTableScanInfo->numOfOutput, + pQuery->current->groupIndex); + } + + (*status) = doFilterByBlockTimeWindow(pTableScanInfo, pBlock); + } + + SDataBlockInfo* pBlockInfo = &pBlock->info; if ((*status) == BLK_DATA_NO_NEEDED) { - qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("QInfo:%"PRIu64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, + pBlockInfo->window.ekey, pBlockInfo->rows); pCost->discardBlocks += 1; } else if ((*status) == BLK_DATA_STATIS_NEEDED) { - // this function never returns error? - tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); pCost->loadBlockStatis += 1; + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); - if (*pStatis == NULL) { // data block statistics does not exist, load data block - *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); - pCost->totalCheckedRows += pBlockInfo->rows; + if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pCost->totalCheckedRows += pBlock->info.rows; } } else { assert((*status) == BLK_DATA_ALL_NEEDED); // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); + + if (pQuery->topBotQuery && pBlock->pBlockStatis != NULL) { + bool load = false; + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pTableScanInfo->pCtx[i].functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockStatis[i].min), + (char*)&(pBlock->pBlockStatis[i].max)); + if (!load) { // current block has been discard due to filter applied + pCost->discardBlocks += 1; + qDebug("QInfo:%"PRIu64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, + pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; + } + } + } + } - if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { - // current block has been discard due to filter applied + // current block has been discard due to filter applied + if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) { pCost->discardBlocks += 1; - qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); + qDebug("QInfo:%"PRIu64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey, + pBlockInfo->window.ekey, pBlockInfo->rows); (*status) = BLK_DATA_DISCARD; + return TSDB_CODE_SUCCESS; } pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; - *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); - if (*pDataBlock == NULL) { + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + if (pBlock->pDataBlock == NULL) { return terrno; } + + doSetFilterColumnInfo(pQuery, pBlock); + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL) { + filterRowsInDataBlock(pRuntimeEnv, pQuery->pFilterInfo, pQuery->numOfFilterCols, pBlock, ascQuery); + } } return TSDB_CODE_SUCCESS; @@ -2716,261 +2735,100 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { return midPos; } -static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capacity) { - SQuery* pQuery = pRuntimeEnv->pQuery; +/* + * set tag value in SQLFunctionCtx + * e.g.,tag information into input buffer + */ +static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) { + tVariantDestroy(tag); - if (capacity < pQuery->rec.capacity) { - return; + char* val = NULL; + if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { + val = tsdbGetTableName(pTable); + assert(val != NULL); + } else { + val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); } - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pQuery->pExpr1[i].bytes; - assert(bytes > 0 && capacity > 0); - - char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage)); - if (tmp == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } else { - pQuery->sdata[i] = (tFilePage *)tmp; - } - - // set the pCtx output buffer position - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data; + if (val == NULL || isNull(val, type)) { + tag->nType = TSDB_DATA_TYPE_NULL; + return; } - qDebug("QInfo:%p realloc output buffer to inc output buffer from: %" PRId64 " rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv), - pQuery->rec.capacity, capacity); - - pQuery->rec.capacity = capacity; -} - -// TODO merge with enuserOutputBufferSimple -static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { - // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block - SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyColumn && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { - SResultRec *pRec = &pQuery->rec; - - if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { - int32_t remain = (int32_t)(pRec->capacity - pRec->rows); - int32_t newSize = (int32_t)(pRec->capacity + (pBlockInfo->rows - remain)); - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t bytes = pQuery->pExpr1[i].bytes; - assert(bytes > 0 && newSize > 0); - - char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); - if (tmp == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } else { - memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); - pQuery->sdata[i] = (tFilePage *)tmp; - } - - // set the pCtx output buffer position - pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes; - - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; - } - } - - qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), - newSize, pRec->capacity, newSize - pRec->rows); - - pRec->capacity = newSize; - } + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + int32_t maxLen = bytes - VARSTR_HEADER_SIZE; + int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); + tVariantCreateFromBinary(tag, varDataVal(val), len, type); + //tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); + } else { + tVariantCreateFromBinary(tag, val, bytes, type); } } -static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) { - STimeWindow w = TSWINDOW_INITIALIZER; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { + assert(pTagColList != NULL && numOfTags > 0); - if (QUERY_IS_ASC_QUERY(pQuery)) { - getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); - pWindowResInfo->prevSKey = w.skey; - } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w); - pWindowResInfo->prevSKey = w.skey; + for(int32_t i = 0; i < numOfTags; ++i) { + if (pTagColList[i].colId == colId) { + return &pTagColList[i]; } } + + return NULL; } -static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - SQueryCostInfo* summary = &pRuntimeEnv->summary; +void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + SQueryRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - qDebug("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", lastkey:%" PRId64 ", order:%d", - GET_QINFO_ADDR(pRuntimeEnv), pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, pTableQueryInfo->lastKey, - pQuery->order.order); + SExprInfo *pExpr = pOperatorInfo->pExpr; + SQuery *pQuery = pRuntimeEnv->pQuery; - TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + SExprInfo* pExprInfo = &pExpr[0]; + if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP && pQuery->stableQuery) { + assert(pExprInfo->base.numOfParams == 1); - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - while (tsdbNextDataBlock(pQueryHandle)) { - summary->totalBlocks += 1; + int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } + doSetTagValueInParam(pTable, tagColId, &pCtx[0].tag, pColInfo->type, pColInfo->bytes); + return; + } else { + // set tag value, by which the results are aggregated. + int32_t offset = 0; + memset(pRuntimeEnv->tagVal, 0, pQuery->tagLen); - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - doSetInitialTimewindow(pRuntimeEnv, &blockInfo); + for (int32_t idx = 0; idx < numOfOutput; ++idx) { + SExprInfo* pLocalExprInfo = &pExpr[idx]; - // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block - ensureOutputBuffer(pRuntimeEnv, &blockInfo); + // ts_comp column required the tag value for join filter + if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { + continue; + } - SDataStatis *pStatis = NULL; - SArray * pDataBlock = NULL; - uint32_t status = 0; + // todo use tag column index to optimize performance + doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->type, + pLocalExprInfo->bytes); - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); - if (ret != TSDB_CODE_SUCCESS) { - break; - } + if (IS_NUMERIC_TYPE(pLocalExprInfo->type) || pLocalExprInfo->type == TSDB_DATA_TYPE_BOOL) { + memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i64, pLocalExprInfo->bytes); + } else { + memcpy(pRuntimeEnv->tagVal + offset, pCtx[idx].tag.pz, pCtx[idx].tag.nLen); + } - if (status == BLK_DATA_DISCARD) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; + offset += pLocalExprInfo->bytes; } - // query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : blockInfo.rows - 1; - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); - - summary->totalRows += blockInfo.rows; - qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), - blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); - - // while the output buffer is full or limit/offset is applied, query may be paused here - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL | QUERY_COMPLETED)) { - break; + //todo : use index to avoid iterator all possible output columns + if (pQuery->stableQuery && pQuery->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddev(pRuntimeEnv, pCtx, numOfOutput, pExprInfo); } } - if (terrno != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, terrno); - } - - // if the result buffer is not full, set the query complete - if (!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - setQueryStatus(pQuery, QUERY_COMPLETED); + // set the tsBuf start position before check each data block + if (pRuntimeEnv->pTsBuf != NULL) { + setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); } - - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - closeAllResultRows(&pRuntimeEnv->windowResInfo); - pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window - } - - return 0; -} - -/* - * set tag value in SQLFunctionCtx - * e.g.,tag information into input buffer - */ -static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVariant *tag, int16_t type, int16_t bytes) { - tVariantDestroy(tag); - - if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { - char* val = tsdbGetTableName(pTable); - assert(val != NULL); - - tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY); - } else { - char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); - if (val == NULL) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - if (isNull(val, type)) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } - - tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); - } else { - if (isNull(val, type)) { - tag->nType = TSDB_DATA_TYPE_NULL; - return; - } - - tVariantCreateFromBinary(tag, val, bytes, type); - } - } -} - -static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { - assert(pTagColList != NULL && numOfTags > 0); - - for(int32_t i = 0; i < numOfTags; ++i) { - if (pTagColList[i].colId == colId) { - return &pTagColList[i]; - } - } - - return NULL; -} - -void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { - SQuery *pQuery = pRuntimeEnv->pQuery; - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - - SExprInfo *pExprInfo = &pQuery->pExpr1[0]; - if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP && pRuntimeEnv->stableQuery) { - assert(pExprInfo->base.numOfParams == 1); - - int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; - SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - - doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); - } else { - // set tag value, by which the results are aggregated. - for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { - SExprInfo* pLocalExprInfo = &pQuery->pExpr1[idx]; - - // ts_comp column required the tag value for join filter - if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { - continue; - } - - // todo use tag column index to optimize performance - doSetTagValueInParam(tsdb, pTable, pLocalExprInfo->base.colInfo.colId, &pRuntimeEnv->pCtx[idx].tag, - pLocalExprInfo->type, pLocalExprInfo->bytes); - } - - // set the join tag for first column - SSqlFuncMsg *pFuncMsg = &pExprInfo->base; - if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTsBuf != NULL && - pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - assert(pFuncMsg->numOfParams == 1); - - int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; - SColumnInfo *pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - - doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); - - int16_t tagType = pRuntimeEnv->pCtx[0].tag.nType; - if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%s", pQInfo, - pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.pz); - } else { - qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, pQInfo, - pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64); - } - } - } -} +} static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t srcDataType) { if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { @@ -3075,253 +2933,36 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim } } -typedef struct SCompSupporter { - STableQueryInfo **pTableQueryInfo; - int32_t *rowIndex; - int32_t order; -} SCompSupporter; - -int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { - int32_t left = *(int32_t *)pLeft; - int32_t right = *(int32_t *)pRight; - - SCompSupporter * supporter = (SCompSupporter *)param; - - int32_t leftPos = supporter->rowIndex[left]; - int32_t rightPos = supporter->rowIndex[right]; - - /* left source is exhausted */ - if (leftPos == -1) { - return 1; - } - - /* right source is exhausted*/ - if (rightPos == -1) { - return -1; - } - - STableQueryInfo** pList = supporter->pTableQueryInfo; - - SResultRowInfo *pWindowResInfo1 = &(pList[left]->windowResInfo); - SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); - TSKEY leftTimestamp = pWindowRes1->win.skey; - - SResultRowInfo *pWindowResInfo2 = &(pList[right]->windowResInfo); - SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); - TSKEY rightTimestamp = pWindowRes2->win.skey; - - if (leftTimestamp == rightTimestamp) { - return 0; - } - - if (supporter->order == TSDB_ORDER_ASC) { - return (leftTimestamp > rightTimestamp)? 1:-1; - } else { - return (leftTimestamp < rightTimestamp)? 1:-1; - } -} - -int32_t mergeGroupResult(SQInfo *pQInfo) { - int64_t st = taosGetTimestampUs(); - - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - while (pQInfo->groupIndex < numOfGroups) { - SArray *group = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); - - int32_t ret = mergeIntoGroupResultImpl(pGroupResInfo, group, pQInfo); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - // this group generates at least one result, return results - pQInfo->groupIndex += 1; - if (taosArrayGetSize(pGroupResInfo->pRows) > 0) { - break; - } - - qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); - taosArrayClear(pGroupResInfo->pRows); - - pGroupResInfo->index = 0; - pGroupResInfo->rowId = 0; - } - - if (pQInfo->groupIndex == numOfGroups && taosArrayGetSize(pGroupResInfo->pRows) == 0) { - SET_STABLE_QUERY_OVER(pQInfo); - } - - int64_t elapsedTime = taosGetTimestampUs() - st; - qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, - pQInfo->groupIndex - 1, numOfGroups, elapsedTime); - - pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; - return TSDB_CODE_SUCCESS; -} - -static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t* index, int32_t orderType); - -void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - - // all results in current group have been returned to client, try next group - if (pGroupResInfo->index >= taosArrayGetSize(pGroupResInfo->pRows)) { - // current results of group has been sent to client, try next group - pGroupResInfo->index = 0; - pGroupResInfo->rowId = 0; - taosArrayClear(pGroupResInfo->pRows); - - if (mergeGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { - return; // failed to save data in the disk - } - - // check if all results has been sent to client - int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (taosArrayGetSize(pGroupResInfo->pRows) == 0 && pQInfo->groupIndex == numOfGroup) { - SET_STABLE_QUERY_OVER(pQInfo); - return; - } - } - - int32_t size = (int32_t) taosArrayGetSize(pGroupResInfo->pRows); - pQuery->rec.rows = doCopyToSData(pQInfo, pGroupResInfo->pRows->pData, (int32_t) size, &pGroupResInfo->index, TSDB_ORDER_ASC); -} - -int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pExpr1[j].base.functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { - continue; - } - - SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j); - assert(pResultInfo != NULL); - - if (pResultInfo->numOfRes > 0) { - return pResultInfo->numOfRes; - } - } - - return 0; -} - -int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery); +void copyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { + SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; + pBlock->info.rows = 0; int32_t code = TSDB_CODE_SUCCESS; - - int32_t *posList = NULL; - SLoserTreeInfo *pTree = NULL; - STableQueryInfo **pTableQueryInfoList = NULL; - - size_t size = taosArrayGetSize(pTableList); - if (pGroupResInfo->pRows == NULL) { - pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); - } - - posList = calloc(size, sizeof(int32_t)); - pTableQueryInfoList = malloc(POINTER_BYTES * size); - - if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL) { - qError("QInfo:%p failed alloc memory", pQInfo); - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _end; - } - - int32_t numOfTables = 0; - for (int32_t i = 0; i < size; ++i) { - STableQueryInfo *item = taosArrayGetP(pTableList, i); - if (item->windowResInfo.size > 0) { - pTableQueryInfoList[numOfTables++] = item; - } - } - - // there is no data in current group - // no need to merge results since only one table in each group - if (numOfTables == 0) { - goto _end; - } - - SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQuery->order.order}; - - int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - if (ret != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _end; - } - - int64_t lastTimestamp = ascQuery? INT64_MIN:INT64_MAX; - int64_t startt = taosGetTimestampMs(); - - while (1) { - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:%p it is already killed, abort", pQInfo); - code = TSDB_CODE_TSC_QUERY_CANCELLED; - goto _end; + while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { + // all results in current group have been returned to client, try next group + if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { + assert(pGroupResInfo->index == 0); + if ((code = mergeIntoGroupResult(&pRuntimeEnv->groupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { + return; + } } - int32_t tableIndex = pTree->pNode[0].index; - - SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->windowResInfo; - SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); - - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); - if (num <= 0) { - cs.rowIndex[tableIndex] += 1; - - if (cs.rowIndex[tableIndex] >= pWindowResInfo->size) { - cs.rowIndex[tableIndex] = -1; - if (--numOfTables == 0) { // all input sources are exhausted - break; - } - } - } else { - assert((pWindowRes->win.skey >= lastTimestamp && ascQuery) || (pWindowRes->win.skey <= lastTimestamp && !ascQuery)); + doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, TSDB_ORDER_ASC, pBlock); - if (pWindowRes->win.skey != lastTimestamp) { - taosArrayPush(pGroupResInfo->pRows, &pWindowRes); - pWindowRes->numOfRows = (uint32_t) num; + // current data are all dumped to result buffer, clear it + if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { + cleanupGroupResInfo(pGroupResInfo); + if (!incNextGroup(pGroupResInfo)) { + break; } + } - lastTimestamp = pWindowRes->win.skey; - - // move to the next row of current entry - if ((++cs.rowIndex[tableIndex]) >= pWindowResInfo->size) { - cs.rowIndex[tableIndex] = -1; - - // all input sources are exhausted - if ((--numOfTables) == 0) { - break; - } + // enough results in data buffer, return + if (pBlock->info.rows >= threshold) { + break; } } - tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries); - } - - int64_t endt = taosGetTimestampMs(); - -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); -#endif - - qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); - - _end: - tfree(pTableQueryInfoList); - tfree(posList); - tfree(pTree); - - return code; } static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) { @@ -3335,71 +2976,17 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * SWITCH_ORDER(pTableQueryInfo->cur.order); pTableQueryInfo->cur.vgroupIndex = -1; - // set the index at the end of time window - pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1; + // set the index to be the end slot of result rows array + pTableQueryInfo->resInfo.curIndex = pTableQueryInfo->resInfo.size - 1; } -static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo *pWindowResInfo, int32_t order) { +static void setupQueryRangeForReverseScan(SQueryRuntimeEnv* pRuntimeEnv) { SQuery* pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - bool closed = getResultRowStatus(pWindowResInfo, i); - if (!closed) { - continue; - } - - SResultRow *pRow = getResultRow(pWindowResInfo, i); - - // open/close the specified query for each group result - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functId = pQuery->pExpr1[j].base.functionId; - SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j); - - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - pInfo->complete = false; - } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - pInfo->complete = true; - } - } - } -} - -void disableFuncInReverseScan(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t order = pQuery->order.order; - - // group by normal columns and interval query on normal table - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { - disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); - } else { // for simple result of table query, - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor - int32_t functId = pQuery->pExpr1[j].base.functionId; - - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[j]; - if (pCtx->resultInfo == NULL) { - continue; // resultInfo is NULL, means no data checked in previous scan - } - - if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || - ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - pCtx->resultInfo->complete = false; - } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - pCtx->resultInfo->complete = true; - } - } - } -} - -static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - + int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pRuntimeEnv)); for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); - SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); + SArray *tableKeyGroup = taosArrayGetP(pQuery->tableGroupInfo.pGroupList, i); size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { @@ -3416,378 +3003,168 @@ static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { } } -void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SWITCH_ORDER(pRuntimeEnv->pCtx[i].order); +void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t i = 0; i < numOfOutput; ++i) { + SWITCH_ORDER(pCtx[i].order); } } int32_t initResultRow(SResultRow *pResultRow) { pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); pResultRow->pageId = -1; - pResultRow->rowId = -1; + pResultRow->offset = -1; return TSDB_CODE_SUCCESS; } -void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; +/* + * The start of each column SResultRowCellInfo is denote by RowCellInfoOffset. + * Note that in case of top/bottom query, the whole multiple rows of result is treated as only one row of results. + * +------------+-----------------result column 1-----------+-----------------result column 2-----------+ + * + SResultRow | SResultRowCellInfo | intermediate buffer1 | SResultRowCellInfo | intermediate buffer 2| + * +------------+-------------------------------------------+-------------------------------------------+ + * offset[0] offset[1] + */ +void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid) { + SQLFunctionCtx* pCtx = pInfo->pCtx; + SSDataBlock* pDataBlock = pInfo->pRes; + int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; + SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; int32_t tid = 0; - int64_t uid = 0; - SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid); + SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&tid, sizeof(tid), true, uid); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->aOutputBuf = pQuery->sdata[i]->data; + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); /* * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. */ - SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i); + SResultRowCellInfo* pCellInfo = getResultCell(pRow, i, rowCellInfoOffset); RESET_RESULT_INFO(pCellInfo); - pCtx->resultInfo = pCellInfo; + + pCtx[i].resultInfo = pCellInfo; + pCtx[i].pOutput = pData->pData; + assert(pCtx[i].pOutput != NULL); // set the timestamp output buffer for top/bottom/diff query - int32_t functionId = pQuery->pExpr1[i].base.functionId; + int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + pCtx[i].ptsOutputBuf = pCtx[0].pOutput; } - - memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pExpr1[i].bytes * pQuery->rec.capacity)); } - initCtxOutputBuf(pRuntimeEnv); + initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); } -void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { - SQuery *pQuery = pRuntimeEnv->pQuery; +void updateOutputBuf(SArithOperatorInfo* pInfo, int32_t numOfInputRows) { + SOptrBasicInfo* pBInfo = &pInfo->binfo; + SSDataBlock* pDataBlock = pBInfo->pRes; - // reset the execution contexts - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pExpr1[j].base.functionId; - assert(functionId != TSDB_FUNC_DIFF); + int32_t newSize = pDataBlock->info.rows + numOfInputRows; + if (pInfo->bufCapacity < newSize) { + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + char* p = realloc(pColInfo->pData, newSize * pColInfo->info.bytes); + if (p != NULL) { + pColInfo->pData = p; - // set next output position - if (IS_OUTER_FORWARD(aAggs[functionId].nStatus)) { - pRuntimeEnv->pCtx[j].aOutputBuf += pRuntimeEnv->pCtx[j].outputBytes * output; + // it starts from the tail of the previously generated results. + pBInfo->pCtx[i].pOutput = pColInfo->pData; + pInfo->bufCapacity = newSize; + } else { + // longjmp + } } + } - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - /* - * NOTE: for top/bottom query, the value of first column of output (timestamp) are assigned - * in the procedure of top/bottom routine - * the output buffer in top/bottom routine is ptsOutputBuf, so we need to forward the output buffer - * - * diff function is handled in multi-output function - */ - pRuntimeEnv->pCtx[j].ptsOutputBuf = (char*)pRuntimeEnv->pCtx[j].ptsOutputBuf + TSDB_KEYSIZE * output; - } + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i); + pBInfo->pCtx[i].pOutput = pColInfo->pData + pColInfo->info.bytes * pDataBlock->info.rows; - RESET_RESULT_INFO(pRuntimeEnv->pCtx[j].resultInfo); + // re-estabilish output buffer pointer. + int32_t functionId = pBInfo->pCtx[i].functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pBInfo->pCtx[i].ptsOutputBuf = pBInfo->pCtx[0].pOutput; + } } } -void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pQuery->pExpr1[j].base.functionId; - pRuntimeEnv->pCtx[j].currentStage = 0; - - SResultRowCellInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); +void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { + for (int32_t j = 0; j < size; ++j) { + SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (pResInfo->initialized) { continue; } - aAggs[functionId].init(&pRuntimeEnv->pCtx[j]); + aAggs[pCtx[j].functionId].init(&pCtx[j]); } } -void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - if (pQuery->rec.rows == 0 || pQuery->limit.offset == 0) { - return; - } - - if (pQuery->rec.rows <= pQuery->limit.offset) { - qDebug("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows, - pQuery->limit.offset - pQuery->rec.rows); - - pQuery->limit.offset -= pQuery->rec.rows; - pQuery->rec.rows = 0; - - resetDefaultResInfoOutputBuf(pRuntimeEnv); - - // clear the buffer full flag if exists - CLEAR_QUERY_STATUS(pQuery, QUERY_RESBUF_FULL); +void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status) { + if (status == QUERY_NOT_COMPLETED) { + pRuntimeEnv->status = status; } else { - int64_t numOfSkip = pQuery->limit.offset; - pQuery->rec.rows -= numOfSkip; - pQuery->limit.offset = 0; + // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first + CLEAR_QUERY_STATUS(pRuntimeEnv, QUERY_NOT_COMPLETED); + pRuntimeEnv->status |= status; + } +} - qDebug("QInfo:%p skip row:%"PRId64", new offset:%d, numOfRows remain:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), numOfSkip, - 0, pQuery->rec.rows); +static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { + SQuery *pQuery = pRuntimeEnv->pQuery; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; + if (pRuntimeEnv->pTsBuf) { + SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } - memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes)); - pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; + // reverse order time range + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { - pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; - } - } + SET_REVERSE_SCAN_FLAG(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); - updateNumOfResult(pRuntimeEnv, (int32_t)pQuery->rec.rows); - } -} + switchCtxOrder(pCtx, numOfOutput); + SWITCH_ORDER(pQuery->order.order); -void setQueryStatus(SQuery *pQuery, int8_t status) { - if (status == QUERY_NOT_COMPLETED) { - pQuery->status = status; - } else { - // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pQuery, QUERY_NOT_COMPLETED); - pQuery->status |= status; - } + setupQueryRangeForReverseScan(pRuntimeEnv); } -bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { +void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; - bool toContinue = false; - if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { + int32_t numOfOutput = pOperator->numOfOutput; + if (pQuery->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery) || pQuery->sw.gap > 0) { // for each group result, call the finalize function for each column - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + if (pQuery->groupbyColumn) { + closeAllResultRows(pResultRowInfo); + } - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pResult = getResultRow(pWindowResInfo, i); - - setResultOutputBuf(pRuntimeEnv, pResult); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int16_t functId = pQuery->pExpr1[j].base.functionId; - if (functId == TSDB_FUNC_TS) { - continue; - } - - aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - - toContinue |= (!pResInfo->complete); - } - } - } else { - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int16_t functId = pQuery->pExpr1[j].base.functionId; - if (functId == TSDB_FUNC_TS) { - continue; - } - - aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - - toContinue |= (!pResInfo->complete); - } - } - - return toContinue; -} - -static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - assert((start <= pTableQueryInfo->lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (start >= pTableQueryInfo->lastKey && !QUERY_IS_ASC_QUERY(pQuery))); - - SQueryStatusInfo info = { - .status = pQuery->status, - .windowIndex = pRuntimeEnv->windowResInfo.curIndex, - .lastKey = start, - }; - - TIME_WINDOW_COPY(info.w, pQuery->window); - return info; -} - -static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { - SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; - - pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); // save the cursor - if (pRuntimeEnv->pTsBuf) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); - bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); - assert(ret); - } - - // reverse order time range - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); - - if (QUERY_IS_ASC_QUERY(pQuery)) { - assert(pQuery->window.skey <= pQuery->window.ekey); - } else { - assert(pQuery->window.skey >= pQuery->window.ekey); - } - - SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - switchCtxOrder(pRuntimeEnv); - disableFuncInReverseScan(pQInfo); - setupQueryRangeForReverseScan(pQInfo); - - // clean unused handle - if (pRuntimeEnv->pSecQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - } - - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - if (pRuntimeEnv->pSecQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } -} - -static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - SWITCH_ORDER(pQuery->order.order); - switchCtxOrder(pRuntimeEnv); - - tsBufSetCursor(pRuntimeEnv->pTsBuf, &pStatus->cur); - if (pRuntimeEnv->pTsBuf) { - pRuntimeEnv->pTsBuf->cur.order = pQuery->order.order; - } - - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - - // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan - pTableQueryInfo->lastKey = pStatus->lastKey; - pQuery->status = pStatus->status; - - pTableQueryInfo->win = pStatus->w; - pQuery->window = pTableQueryInfo->win; -} - -static void restoreTimeWindow(STableGroupInfo* pTableGroupInfo, STsdbQueryCond* pCond) { - assert(pTableGroupInfo->numOfTables == 1); - SArray* pTableKeyGroup = taosArrayGetP(pTableGroupInfo->pGroupList, 0); - STableKeyInfo* pKeyInfo = taosArrayGet(pTableKeyGroup, 0); - pKeyInfo->lastKey = pCond->twindow.skey; -} - -void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { - SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo *pTableQueryInfo = pQuery->current; - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - - // store the start query position - SQueryStatusInfo qstatus = getQueryStatusInfo(pRuntimeEnv, start); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); - - if (!pRuntimeEnv->groupbyColumn && pRuntimeEnv->hasTagResults) { - setTagVal(pRuntimeEnv, pTableQueryInfo->pTable, pQInfo->tsdb); - } - - while (1) { - doScanAllDataBlocks(pRuntimeEnv); - - if (pRuntimeEnv->scanFlag == MASTER_SCAN) { - qstatus.status = pQuery->status; - - // do nothing if no data blocks are found qualified during scan - if (qstatus.lastKey == pTableQueryInfo->lastKey) { - qDebug("QInfo:%p no results generated in this scan", pQInfo); - } - } - - if (!needScanDataBlocksAgain(pRuntimeEnv)) { - // restore the status code and jump out of loop - if (pRuntimeEnv->scanFlag == REPEAT_SCAN) { - pQuery->status = qstatus.status; - } - - break; - } - - if (pRuntimeEnv->pSecQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - } - - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); - restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - if (pRuntimeEnv->pSecQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } - - pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - pRuntimeEnv->scanFlag = REPEAT_SCAN; - - qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, - cond.twindow.skey, cond.twindow.ekey); - - // check if query is killed or not - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - } - - if (!needReverseScan(pQuery)) { - return; - } - - setEnvBeforeReverseScan(pRuntimeEnv, &qstatus); - - // reverse scan from current position - qDebug("QInfo:%p start to reverse scan", pQInfo); - doScanAllDataBlocks(pRuntimeEnv); - - clearEnvAfterReverseScan(pRuntimeEnv, &qstatus); -} - -void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) { - // for each group result, call the finalize function for each column - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pRuntimeEnv->groupbyColumn) { - closeAllResultRows(pWindowResInfo); - } - - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *buf = pWindowResInfo->pResult[i]; - if (!isResultRowClosed(pWindowResInfo, i)) { + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *buf = pResultRowInfo->pResult[i]; + if (!isResultRowClosed(pResultRowInfo, i)) { continue; } - setResultOutputBuf(pRuntimeEnv, buf); + setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - aAggs[pQuery->pExpr1[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + for (int32_t j = 0; j < numOfOutput; ++j) { + aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); } /* * set the number of output results for group by normal columns, the number of output rows usually is 1 except * the top and bottom query */ - buf->numOfRows = (uint16_t)getNumOfResult(pRuntimeEnv); + buf->numOfRows = (uint16_t)getNumOfResult(pRuntimeEnv, pCtx, numOfOutput); } } else { - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - aAggs[pQuery->pExpr1[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); + for (int32_t j = 0; j < numOfOutput; ++j) { + aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]); } } } @@ -3804,9 +3181,7 @@ static bool hasMainOutput(SQuery *pQuery) { return false; } -static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void* pTable, STimeWindow win, void* buf) { - SQuery *pQuery = pRuntimeEnv->pQuery; - +static STableQueryInfo *createTableQueryInfo(SQuery* pQuery, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { STableQueryInfo *pTableQueryInfo = buf; pTableQueryInfo->win = win; @@ -3816,9 +3191,9 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void pTableQueryInfo->cur.vgroupIndex = -1; // set more initial size of interval/groupby query - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { + if (QUERY_IS_INTERVAL_QUERY(pQuery) || groupbyColumn) { int32_t initialSize = 128; - int32_t code = initResultRowInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT); + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -3834,33 +3209,43 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { } tVariantDestroy(&pTableQueryInfo->tag); - cleanupResultRowInfo(&pTableQueryInfo->windowResInfo); + cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -/** - * set output buffer for different group - * @param pRuntimeEnv - * @param pDataBlockInfo - */ -void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; +void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset) { + // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - // lastKey needs to be updated - pTableQueryInfo->lastKey = nextKey; + int16_t offset = 0; + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { - setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); - } + SResultRowCellInfo* pResInfo = pCtx[i].resultInfo; + if (pResInfo->initialized && pResInfo->complete) { + offset += pCtx[i].outputBytes; + continue; + } - if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == groupIndex) { - return; + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQuery, bufPage, pResult->offset, offset); + offset += pCtx[i].outputBytes; + + int32_t functionId = pCtx[i].functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { + pCtx[i].ptsOutputBuf = pCtx[0].pOutput; + } + + if (!pResInfo->initialized) { + aAggs[functionId].init(&pCtx[i]); + } } +} +void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, + int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t groupIndex) { int64_t uid = 0; - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, - sizeof(groupIndex), true, uid); + SResultRow* pResultRow = + doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char*)&groupIndex, sizeof(groupIndex), true, uid); assert (pResultRow != NULL); /* @@ -3868,116 +3253,191 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pResultRow->pageId == -1) { - if (addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != - TSDB_CODE_SUCCESS) { + int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->pQuery->resultRowSize); + if (ret != TSDB_CODE_SUCCESS) { return; } } + setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); +} + +void setExecutionContext(SQueryRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t groupIndex, + TSKEY nextKey) { + STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; + + // lastKey needs to be updated + pTableQueryInfo->lastKey = nextKey; + if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == groupIndex) { + return; + } + + doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, groupIndex); + // record the current active group id pRuntimeEnv->prevGroupId = groupIndex; - setResultOutputBuf(pRuntimeEnv, pResultRow); - initCtxOutputBuf(pRuntimeEnv); } -void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { - SQuery *pQuery = pRuntimeEnv->pQuery; - +void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, + int32_t numOfCols, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, page); + int16_t offset = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + pCtx[i].pOutput = getPosInResultPage(pRuntimeEnv->pQuery, page, pResult->offset, offset); + offset += pCtx[i].outputBytes; - int32_t functionId = pQuery->pExpr1[i].base.functionId; + int32_t functionId = pCtx[i].functionId; if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + pCtx[i].ptsOutputBuf = pCtx[0].pOutput; } /* * set the output buffer information and intermediate buffer, * not all queries require the interResultBuf, such as COUNT */ - pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); } } -void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { - SQuery *pQuery = pRuntimeEnv->pQuery; +void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { + SQuery* pQuery = pRuntimeEnv->pQuery; - // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); + SSqlFuncMsg* pFuncMsg = &pExprInfo->base; + if (pQuery->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && + (pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && + (pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX)) { + assert(pFuncMsg->numOfParams == 1); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; + int16_t tagColId = (int16_t)pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); - if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) { - continue; + doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); + + int16_t tagType = pCtx[0].tag.nType; + if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%"PRIu64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), + pExprInfo->base.arg->argValue.i64, pCtx[0].tag.pz); + } else { + qDebug("QInfo:%"PRIu64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), + pExprInfo->base.arg->argValue.i64, pCtx[0].tag.i64); } + } +} - pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, bufPage); - pCtx->currentStage = 0; +int32_t setTimestampListJoinInfo(SQueryRuntimeEnv* pRuntimeEnv, tVariant* pTag, STableQueryInfo *pTableQueryInfo) { + SQuery* pQuery = pRuntimeEnv->pQuery; - int32_t functionId = pCtx->functionId; - if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) { - pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; + assert(pRuntimeEnv->pTsBuf != NULL); + + // both the master and supplement scan needs to set the correct ts comp start position + if (pTableQueryInfo->cur.vgroupIndex == -1) { + tVariantAssign(&pTableQueryInfo->tag, pTag); + + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQuery->vgId, &pTableQueryInfo->tag); + + // failed to find data with the specified tag value and vnodeId + if (!tsBufIsValidElem(&elem)) { + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qError("QInfo:%"PRIu64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); + } else { + qError("QInfo:%"PRIu64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i64); + } + + return -1; + } + + // Keep the cursor info of current table + pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%"PRIu64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + qDebug("QInfo:%"PRIu64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } - if (!pCtx->resultInfo->initialized) { - aAggs[functionId].init(pCtx); + } else { + tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%"PRIu64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + } else { + qDebug("QInfo:%"PRIu64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } -} -int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQueryInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + return 0; +} - setTagVal(pRuntimeEnv, pTable, pQInfo->tsdb); +void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr) { + SQuery* pQuery = pRuntimeEnv->pQuery; - // both the master and supplement scan needs to set the correct ts comp start position - if (pRuntimeEnv->pTsBuf != NULL) { - tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; + int32_t numOfExprs = pQuery->numOfOutput; + for(int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExprInfo = &(pExpr[i]); + if (pExprInfo->base.functionId != TSDB_FUNC_STDDEV_DST) { + continue; + } - if (pTableQueryInfo->cur.vgroupIndex == -1) { - tVariantAssign(&pTableQueryInfo->tag, pTag); + SSqlFuncMsg* pFuncMsg = &pExprInfo->base; - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, &pTableQueryInfo->tag); + pCtx[i].param[0].arr = NULL; + pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); - } else { - qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64); + // TODO use hash to speedup this loop + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t j = 0; j < numOfGroup; ++j) { + SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); + if (pQuery->tagLen == 0 || memcmp(p->tags, pRuntimeEnv->tagVal, pQuery->tagLen) == 0) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t k = 0; k < numOfCols; ++k) { + SStddevInterResult* pres = taosArrayGet(p->pResult, k); + if (pres->colId == pFuncMsg->colInfo.colId) { + pCtx[i].param[0].arr = pres->pResult; + break; + } } - - return false; } + } + } - // keep the cursor info of current meter - pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } +} - } else { - tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); +void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { + SQuery* pQuery = pRuntimeEnv->pQuery; - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + int32_t numOfExprs = pQuery->numOfOutput; + for(int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExprInfo = &(pExpr[i]); + if (pExprInfo->base.functionId != TSDB_FUNC_STDDEV_DST) { + continue; + } + + SSqlFuncMsg* pFuncMsg = &pExprInfo->base; + + pCtx[i].param[0].arr = NULL; + pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + + // TODO use hash to speedup this loop + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t j = 0; j < numOfGroup; ++j) { + SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); + if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t k = 0; k < numOfCols; ++k) { + SStddevInterResult* pres = taosArrayGet(p->pResult, k); + if (pres->colId == pFuncMsg->colInfo.colId) { + pCtx[i].param[0].arr = pres->pResult; + break; + } + } } } } - return 0; } + + /* * There are two cases to handle: * @@ -3987,158 +3447,128 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there * is a previous result generated or not. */ -void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - STableQueryInfo *pTableQueryInfo = pQuery->current; - - if (pTableQueryInfo->queryRangeSet) { - pTableQueryInfo->lastKey = key; - } else { - pTableQueryInfo->win.skey = key; - STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; +void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) { + SQuery *pQuery = pRuntimeEnv->pQuery; + STableQueryInfo *pTableQueryInfo = pQuery->current; + SResultRowInfo *pWindowResInfo = &pTableQueryInfo->resInfo; - // for too small query range, no data in this interval. - if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey < pQuery->window.skey)) || - (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey < pQuery->window.ekey))) { - return; - } + if (pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL) { + return; + } - /** - * In handling the both ascending and descending order super table query, we need to find the first qualified - * timestamp of this table, and then set the first qualified start timestamp. - * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional - * operations involve. - */ - STimeWindow w = TSWINDOW_INITIALIZER; - SResultRowInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; + pTableQueryInfo->win.skey = key; + STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; - TSKEY sk = MIN(win.skey, win.ekey); - TSKEY ek = MAX(win.skey, win.ekey); - getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w); + /** + * In handling the both ascending and descending order super table query, we need to find the first qualified + * timestamp of this table, and then set the first qualified start timestamp. + * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional + * operations involve. + */ + STimeWindow w = TSWINDOW_INITIALIZER; - if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { - if (!QUERY_IS_ASC_QUERY(pQuery)) { - assert(win.ekey == pQuery->window.ekey); - } + TSKEY sk = MIN(win.skey, win.ekey); + TSKEY ek = MAX(win.skey, win.ekey); + getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w); - pWindowResInfo->prevSKey = w.skey; + if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { + if (!QUERY_IS_ASC_QUERY(pQuery)) { + assert(win.ekey == pQuery->window.ekey); } - pTableQueryInfo->queryRangeSet = 1; - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + pWindowResInfo->prevSKey = w.skey; } -} -bool requireTimestamp(SQuery *pQuery) { - for (int32_t i = 0; i < pQuery->numOfOutput; i++) { - int32_t functionId = pQuery->pExpr1[i].base.functionId; - if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_NEED_TS) != 0) { - return true; - } - } - return false; + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; } -bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { - /* - * 1. if skey or ekey locates in this block, we need to load the timestamp column to decide the precise position - * 2. if there are top/bottom, first_dst/last_dst functions, we need to load timestamp column in any cases; - */ - STimeWindow *w = &pDataBlockInfo->window; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - bool loadPrimaryTS = (pTableQueryInfo->lastKey >= w->skey && pTableQueryInfo->lastKey <= w->ekey) || - (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); +/** + * copyToOutputBuf support copy data in ascending/descending order + * For interval query of both super table and table, copy the data in ascending order, since the output results are + * ordered in SWindowResutl already. While handling the group by query for both table and super table, + * all group result are completed already. + * + * @param pQInfo + * @param result + */ - return loadPrimaryTS; -} +static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { + SQuery *pQuery = pRuntimeEnv->pQuery; -static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t *index, int32_t orderType) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + int32_t numOfResult = pBlock->info.rows; // there are already exists result rows - int32_t numOfResult = 0; int32_t start = 0; int32_t step = -1; - qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); + qDebug("QInfo:%"PRIu64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); if (orderType == TSDB_ORDER_ASC) { - start = (*index); + start = pGroupResInfo->index; step = 1; } else { // desc order copy all data - start = numOfRows - (*index) - 1; + start = numOfRows - pGroupResInfo->index - 1; step = -1; } - SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; - for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { - if (pRows[i]->numOfRows == 0) { - (*index) += 1; - pGroupResInfo->rowId = 0; + SResultRow* pRow = taosArrayGetP(pGroupResInfo->pRows, i); + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; continue; } - int32_t numOfRowsToCopy = pRows[i]->numOfRows - pGroupResInfo->rowId; - int32_t oldOffset = pGroupResInfo->rowId; + int32_t numOfRowsToCopy = pRow->numOfRows; - /* - * current output space is not enough to accommodate all data of this page, only partial results - * will be copied to SQuery object's result buffer - */ - if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { - numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; - pGroupResInfo->rowId += numOfRowsToCopy; - } else { - pGroupResInfo->rowId = 0; - (*index) += 1; - } + pGroupResInfo->index += 1; - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRows[i]->pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t size = pRuntimeEnv->pCtx[j].outputBytes; + int16_t offset = 0; + for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j); + int32_t bytes = pColInfoData->info.bytes; - char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, pRows[i], page); - memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); + char *out = pColInfoData->pData + numOfResult * bytes; + char *in = getPosInResultPage(pQuery, page, pRow->offset, offset); + memcpy(out, in, bytes * numOfRowsToCopy); + + offset += bytes; } numOfResult += numOfRowsToCopy; - if (numOfResult == pQuery->rec.capacity) { + if (numOfResult == pRuntimeEnv->resultInfo.capacity) { // output buffer is full break; } } - qDebug("QInfo:%p copy data to query buf completed", pQInfo); - -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, numOfResult); -#endif - return numOfResult; + qDebug("QInfo:%"PRIu64" copy data to query buf completed", GET_QID(pRuntimeEnv)); + pBlock->info.rows = numOfResult; + return 0; } -/** - * copyFromWindowResToSData support copy data in ascending/descending order - * For interval query of both super table and table, copy the data in ascending order, since the output results are - * ordered in SWindowResutl already. While handling the group by query for both table and super table, - * all group result are completed already. - * - * @param pQInfo - * @param result - */ -void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; +static void toSSDataBlock(SGroupResInfo *pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { + assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); + + pBlock->info.rows = 0; + if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { + return; + } + SQuery* pQuery = pRuntimeEnv->pQuery; int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo->pResult, pResultInfo->size, &pQInfo->groupIndex, orderType); + doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); - pQuery->rec.rows += numOfResult; - assert(pQuery->rec.rows <= pQuery->rec.capacity); -} + SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); -static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { + if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + STimeWindow* w = &pBlock->info.window; + w->skey = *(int64_t*)pInfoData->pData; + w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pBlock->info.rows - 1)); + } +} + +static void updateNumOfRowsInResultRows(SQueryRuntimeEnv *pRuntimeEnv, + SQLFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { SQuery *pQuery = pRuntimeEnv->pQuery; // update the number of result for each, only update the number of rows for the corresponding window result. @@ -4146,110 +3576,47 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { return; } - for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - SResultRow *pResult = pRuntimeEnv->windowResInfo.pResult[i]; + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *pResult = pResultRowInfo->pResult[i]; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - int32_t functionId = pRuntimeEnv->pCtx[j].functionId; + for (int32_t j = 0; j < numOfOutput; ++j) { + int32_t functionId = pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { continue; } - SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + SResultRowCellInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } } -static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, - SArray *pDataBlock, __block_search_fn_t searchFn) { - SQuery * pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - SResultRowInfo * pResultRowInfo = &pTableQueryInfo->windowResInfo; - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; - - if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyColumn) { - rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock); - } else { - blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock); - } - - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery), pRuntimeEnv->timeWindowInterpo); - } -} - -bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { +static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pRuntimeEnv->pQuery; - SFillInfo *pFillInfo = pRuntimeEnv->pFillInfo; - - if (pQuery->limit.limit > 0 && pQuery->rec.total >= pQuery->limit.limit) { - return false; - } - - if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { - // There are results not returned to client yet, so filling applied to the remain result is required firstly. - int32_t remain = taosNumOfRemainRows(pFillInfo); - if (remain > 0) { - return true; - } - - /* - * While the code reaches here, there are no results remains now. - * If query is not completed yet, the gaps between two results blocks need to be handled after next data block - * is retrieved from TSDB. - * - * NOTE: If the result set is not the first block, the gap in front of the result set will be filled. If the result - * set is the FIRST result block, the gap between the start time of query time window and the timestamp of the - * first result row in the actual result set will fill nothing. - */ - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - int32_t numOfTotal = (int32_t)getNumOfResWithFill(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity); - return numOfTotal > 0; - } - - } else { - // there are results waiting for returned to client. - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED) && - (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery)) && - (pRuntimeEnv->windowResInfo.size > 0)) { - return true; - } - } - - return false; -} - -static int16_t getNumOfFinalResCol(SQuery* pQuery) { - return pQuery->pExpr2 == NULL? pQuery->numOfOutput:pQuery->numOfExpr2; -} -static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + SSDataBlock* pRes = pRuntimeEnv->outputBuf; if (pQuery->pExpr2 == NULL) { for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - int32_t bytes = pQuery->pExpr1[col].bytes; - - memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); - data += bytes * numOfRows; + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); + data += pColRes->info.bytes * pRes->info.rows; } } else { for (int32_t col = 0; col < pQuery->numOfExpr2; ++col) { - int32_t bytes = pQuery->pExpr2[col].bytes; - - memmove(data, pQuery->sdata[col]->data, bytes * numOfRows); - data += bytes * numOfRows; + SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, col); + memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows); + data += pColRes->info.bytes * numOfRows; } } - int32_t numOfTables = (int32_t) taosHashGetSize(pQInfo->arrTableIdInfo); + int32_t numOfTables = (int32_t) taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); int32_t total = 0; - STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL); + STableIdInfo* item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, NULL); while(item) { STableIdInfo* pDst = (STableIdInfo*)data; @@ -4260,329 +3627,293 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data data += sizeof(STableIdInfo); total++; - qDebug("QInfo:%p set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo, item->tid, item->uid, item->key); - item = taosHashIterate(pQInfo->arrTableIdInfo, item); + qDebug("QInfo:%"PRIu64" set subscribe info, tid:%d, uid:%"PRIu64", skey:%"PRId64, pQInfo->qId, item->tid, item->uid, item->key); + item = taosHashIterate(pRuntimeEnv->pTableRetrieveTsMap, item); } - qDebug("QInfo:%p set %d subscribe info", pQInfo, total); - + qDebug("QInfo:%"PRIu64" set %d subscribe info", pQInfo->qId, total); // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - if (pQInfo->runtimeEnv.stableQuery) { - if (IS_STASBLE_QUERY_OVER(pQInfo)) { - setQueryStatus(pQuery, QUERY_OVER); - } - } else { - if (!queryHasRemainResForTableQuery(&pQInfo->runtimeEnv)) { - setQueryStatus(pQuery, QUERY_OVER); - } - } + if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { + setQueryStatus(pRuntimeEnv, QUERY_OVER); } } -int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int32_t *numOfFilled) { - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; - SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; - - while (1) { - int32_t ret = (int32_t)taosFillResultDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity); - - // todo apply limit output function - /* reached the start position of according to offset value, return immediately */ - if (pQuery->limit.offset == 0) { - qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows", pQInfo, pFillInfo->numOfRows, ret); - return ret; - } - - if (pQuery->limit.offset < ret) { - qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d", - pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0); - - ret -= (int32_t)pQuery->limit.offset; - // todo !!!!there exactly number of interpo is not valid. - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - memmove(pDst[i]->data, pDst[i]->data + pQuery->pExpr1[i].bytes * pQuery->limit.offset, - ret * pQuery->pExpr1[i].bytes); - } - - pQuery->limit.offset = 0; - return ret; - } else { - qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, " - "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0, - pQuery->limit.offset - ret); - - pQuery->limit.offset -= ret; - pQuery->rec.rows = 0; - ret = 0; - } - - if (!queryHasRemainResForTableQuery(pRuntimeEnv)) { - return ret; - } +int32_t doFillTimeIntervalGapsInResults(SFillInfo* pFillInfo, SSDataBlock *pOutput, int32_t capacity) { + void** p = calloc(pFillInfo->numOfCols, POINTER_BYTES); + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pOutput->pDataBlock, i); + p[i] = pColInfoData->pData; } + + pOutput->info.rows = (int32_t)taosFillResultDataBlock(pFillInfo, p, capacity); + tfree(p); + return pOutput->info.rows; } -static void queryCostStatis(SQInfo *pQInfo) { +void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQueryCostInfo *pSummary = &pRuntimeEnv->summary; + SQueryCostInfo *pSummary = &pQInfo->summary; uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); - hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map); + hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); pSummary->hashSize = hashSize; // add the merge time pSummary->elapsedTime += pSummary->firstStageMergeTime; SResultRowPool* p = pQInfo->runtimeEnv.pool; - pSummary->winInfoSize = getResultRowPoolMemSize(p); - pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); + if (p != NULL) { + pSummary->winInfoSize = getResultRowPoolMemSize(p); + pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); + } else { + pSummary->winInfoSize = 0; + pSummary->numOfTimeWindows = 0; + } - qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " + qDebug("QInfo:%"PRIu64" :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, - pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, + pQInfo->qId, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo, pSummary->winInfoSize/1024.0, + qDebug("QInfo:%"PRIu64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); } -static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - STableQueryInfo* pTableQueryInfo = pQuery->current; - - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed - pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; - pQuery->limit.offset = 0; - return; - } - - if (QUERY_IS_ASC_QUERY(pQuery)) { - pQuery->pos = (int32_t)pQuery->limit.offset; - } else { - pQuery->pos = pBlockInfo->rows - (int32_t)pQuery->limit.offset - 1; - } - - assert(pQuery->pos >= 0 && pQuery->pos <= pBlockInfo->rows - 1); - - SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - - // update the pQuery->limit.offset value, and pQuery->pos value - TSKEY *keys = (TSKEY *) pColInfoData->pData; - - // update the offset value - pTableQueryInfo->lastKey = keys[pQuery->pos]; - pQuery->limit.offset = 0; - - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); - - qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), - pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); -} - -void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0) { - return; - } - - pQuery->pos = 0; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - STableQueryInfo* pTableQueryInfo = pQuery->current; - TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; - - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - while (tsdbNextDataBlock(pQueryHandle)) { - if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - - if (pQuery->limit.offset > blockInfo.rows) { - pQuery->limit.offset -= blockInfo.rows; - pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; - pTableQueryInfo->lastKey += step; - - qDebug("QInfo:%p skip rows:%d, offset:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.rows, - pQuery->limit.offset); - } else { // find the appropriated start position in current block - updateOffsetVal(pRuntimeEnv, &blockInfo); - break; - } - } - - if (terrno != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, terrno); - } -} - -static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { - SQuery *pQuery = pRuntimeEnv->pQuery; - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - - assert(pQuery->limit.offset == 0); - STimeWindow tw = *win; - getNextTimeWindow(pQuery, &tw); - - if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || - (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { - - // load the data block and check data remaining in current data block - // TODO optimize performance - SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - - tw = *win; - int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); - assert(startPos >= 0); - - // set the abort info - pQuery->pos = startPos; - - // reset the query start timestamp - pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; - pQuery->window.skey = pTableQueryInfo->win.skey; - TSKEY key = pTableQueryInfo->win.skey; - - pWindowResInfo->prevSKey = tw.skey; - int32_t index = pRuntimeEnv->windowResInfo.curIndex; - - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); - pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index - - qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, - GET_QINFO_ADDR(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, - pQuery->current->lastKey); - - return key; - } else { // do nothing - pQuery->window.skey = tw.skey; - pWindowResInfo->prevSKey = tw.skey; - pTableQueryInfo->lastKey = tw.skey; - - return tw.skey; - } - - return true; -} - -static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { - SQuery *pQuery = pRuntimeEnv->pQuery; - if (QUERY_IS_ASC_QUERY(pQuery)) { - assert(*start <= pQuery->current->lastKey); - } else { - assert(*start >= pQuery->current->lastKey); - } - - // if queried with value filter, do NOT forward query start position - if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { - return true; - } - - /* - * 1. for interval without interpolation query we forward pQuery->interval.interval at a time for - * pQuery->limit.offset times. Since hole exists, pQuery->interval.interval*pQuery->limit.offset value is - * not valid. otherwise, we only forward pQuery->limit.offset number of points - */ - assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); - - STimeWindow w = TSWINDOW_INITIALIZER; - bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); - - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - STableQueryInfo *pTableQueryInfo = pQuery->current; - - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { - tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); - - if (QUERY_IS_ASC_QUERY(pQuery)) { - if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); - pWindowResInfo->prevSKey = w.skey; - } - } else { - getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); - pWindowResInfo->prevSKey = w.skey; - } - - // the first time window - STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); - - while (pQuery->limit.offset > 0) { - STimeWindow tw = win; - - if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { - pQuery->limit.offset -= 1; - pWindowResInfo->prevSKey = win.skey; - - // current time window is aligned with blockInfo.window.ekey - // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; - if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { - pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - } - } - - if (pQuery->limit.offset == 0) { - *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); - return true; - } - - // current window does not ended in current data block, try next data block - getNextTimeWindow(pQuery, &tw); - - /* - * If the next time window still starts from current data block, - * load the primary timestamp column first, and then find the start position for the next queried time window. - * Note that only the primary timestamp column is required. - * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required - * time window resides in current data block. - */ - if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { - - SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); - SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); - - if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { - pQuery->limit.offset -= 1; - } - - if (pQuery->limit.offset == 0) { - *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); - return true; - } else { - tw = win; - int32_t startPos = - getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); - assert(startPos >= 0); - - // set the abort info - pQuery->pos = startPos; - pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; - pWindowResInfo->prevSKey = tw.skey; - win = tw; - } - } else { - break; // offset is not 0, and next time window begins or ends in the next block. - } - } - } - - // check for error - if (terrno != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, terrno); - } - - return true; -} +//static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { +// SQuery *pQuery = pRuntimeEnv->pQuery; +// STableQueryInfo* pTableQueryInfo = pQuery->current; +// +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); +// +// if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed +// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->window.ekey + step : pBlockInfo->window.skey + step; +// pQuery->limit.offset = 0; +// return; +// } +// +// if (QUERY_IS_ASC_QUERY(pQuery)) { +// pQuery->pos = (int32_t)pQuery->limit.offset; +// } else { +// pQuery->pos = pBlockInfo->rows - (int32_t)pQuery->limit.offset - 1; +// } +// +// assert(pQuery->pos >= 0 && pQuery->pos <= pBlockInfo->rows - 1); +// +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// // update the pQuery->limit.offset value, and pQuery->pos value +// TSKEY *keys = (TSKEY *) pColInfoData->pData; +// +// // update the offset value +// pTableQueryInfo->lastKey = keys[pQuery->pos]; +// pQuery->limit.offset = 0; +// +// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); +// +// qDebug("QInfo:%"PRIu64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), +// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); +//} + +//void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { +// SQuery *pQuery = pRuntimeEnv->pQuery; +// +// if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0) { +// return; +// } +// +// pQuery->pos = 0; +// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); +// +// STableQueryInfo* pTableQueryInfo = pQuery->current; +// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// +// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +// while (tsdbNextDataBlock(pQueryHandle)) { +// if (isQueryKilled(pRuntimeEnv->qinfo)) { +// longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); +// } +// +// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// +// if (pQuery->limit.offset > blockInfo.rows) { +// pQuery->limit.offset -= blockInfo.rows; +// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQuery)) ? blockInfo.window.ekey : blockInfo.window.skey; +// pTableQueryInfo->lastKey += step; +// +// qDebug("QInfo:%"PRIu64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, +// pQuery->limit.offset); +// } else { // find the appropriated start position in current block +// updateOffsetVal(pRuntimeEnv, &blockInfo); +// break; +// } +// } +// +// if (terrno != TSDB_CODE_SUCCESS) { +// longjmp(pRuntimeEnv->env, terrno); +// } +//} + +//static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { +// SQuery *pQuery = pRuntimeEnv->pQuery; +// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +// +// assert(pQuery->limit.offset == 0); +// STimeWindow tw = *win; +// getNextTimeWindow(pQuery, &tw); +// +// if ((tw.skey <= pBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || +// (tw.ekey >= pBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQuery))) { +// +// // load the data block and check data remaining in current data block +// // TODO optimize performance +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// tw = *win; +// int32_t startPos = +// getNextQualifiedWindow(pQuery, &tw, pBlockInfo, pColInfoData->pData, binarySearchForKey, -1); +// assert(startPos >= 0); +// +// // set the abort info +// pQuery->pos = startPos; +// +// // reset the query start timestamp +// pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; +// pQuery->window.skey = pTableQueryInfo->win.skey; +// TSKEY key = pTableQueryInfo->win.skey; +// +// pWindowResInfo->prevSKey = tw.skey; +// int32_t index = pRuntimeEnv->resultRowInfo.curIndex; +// +// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); +// pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index +// +// qDebug("QInfo:%"PRIu64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, +// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, +// pQuery->current->lastKey); +// +// return key; +// } else { // do nothing +// pQuery->window.skey = tw.skey; +// pWindowResInfo->prevSKey = tw.skey; +// pTableQueryInfo->lastKey = tw.skey; +// +// return tw.skey; +// } +// +// return true; +//} + +//static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { +// SQuery *pQuery = pRuntimeEnv->pQuery; +// if (QUERY_IS_ASC_QUERY(pQuery)) { +// assert(*start <= pQuery->current->lastKey); +// } else { +// assert(*start >= pQuery->current->lastKey); +// } +// +// // if queried with value filter, do NOT forward query start position +// if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { +// return true; +// } +// +// /* +// * 1. for interval without interpolation query we forward pQuery->interval.interval at a time for +// * pQuery->limit.offset times. Since hole exists, pQuery->interval.interval*pQuery->limit.offset value is +// * not valid. otherwise, we only forward pQuery->limit.offset number of points +// */ +// assert(pRuntimeEnv->resultRowInfo.prevSKey == TSKEY_INITIAL_VAL); +// +// STimeWindow w = TSWINDOW_INITIALIZER; +// bool ascQuery = QUERY_IS_ASC_QUERY(pQuery); +// +// SResultRowInfo *pWindowResInfo = &pRuntimeEnv->resultRowInfo; +// STableQueryInfo *pTableQueryInfo = pQuery->current; +// +// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// +// if (QUERY_IS_ASC_QUERY(pQuery)) { +// if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { +// getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); +// pWindowResInfo->prevSKey = w.skey; +// } +// } else { +// getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); +// pWindowResInfo->prevSKey = w.skey; +// } +// +// // the first time window +// STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery); +// +// while (pQuery->limit.offset > 0) { +// STimeWindow tw = win; +// +// if ((win.ekey <= blockInfo.window.ekey && ascQuery) || (win.ekey >= blockInfo.window.skey && !ascQuery)) { +// pQuery->limit.offset -= 1; +// pWindowResInfo->prevSKey = win.skey; +// +// // current time window is aligned with blockInfo.window.ekey +// // restart it from next data block by set prevSKey to be TSKEY_INITIAL_VAL; +// if ((win.ekey == blockInfo.window.ekey && ascQuery) || (win.ekey == blockInfo.window.skey && !ascQuery)) { +// pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; +// } +// } +// +// if (pQuery->limit.offset == 0) { +// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); +// return true; +// } +// +// // current window does not ended in current data block, try next data block +// getNextTimeWindow(pQuery, &tw); +// +// /* +// * If the next time window still starts from current data block, +// * load the primary timestamp column first, and then find the start position for the next queried time window. +// * Note that only the primary timestamp column is required. +// * TODO: Optimize for this cases. All data blocks are not needed to be loaded, only if the first actually required +// * time window resides in current data block. +// */ +// if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { +// +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); +// +// if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { +// pQuery->limit.offset -= 1; +// } +// +// if (pQuery->limit.offset == 0) { +// *start = doSkipIntervalProcess(pRuntimeEnv, &win, &blockInfo, pTableQueryInfo); +// return true; +// } else { +// tw = win; +// int32_t startPos = +// getNextQualifiedWindow(pQuery, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1); +// assert(startPos >= 0); +// +// // set the abort info +// pQuery->pos = startPos; +// pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos]; +// pWindowResInfo->prevSKey = tw.skey; +// win = tw; +// } +// } else { +// break; // offset is not 0, and next time window begins or ends in the next block. +// } +// } +// } +// +// // check for error +// if (terrno != TSDB_CODE_SUCCESS) { +// longjmp(pRuntimeEnv->env, terrno); +// } +// +// return true; +//} static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); @@ -4590,40 +3921,40 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + // TODO set the tags scan handle if (onlyQueryTags(pQuery)) { return TSDB_CODE_SUCCESS; } - if (isSTableQuery && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isFixedOutputQuery(pRuntimeEnv))) { - return TSDB_CODE_SUCCESS; - } - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + if (isTsCompQuery(pQuery) || isPointInterpoQuery(pQuery)) { + cond.type = BLOCK_LOAD_TABLE_SEQ_ORDER; + } if (!isSTableQuery - && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) + && (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 1) && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) - && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) - && (!isFixedOutputQuery(pRuntimeEnv)) + && (!isGroupbyColumn(pQuery->pGroupbyExpr)) + && (!isFixedOutputQuery(pQuery)) ) { - SArray* pa = GET_TABLEGROUP(pQInfo, 0); + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); STableQueryInfo* pCheckInfo = taosArrayGetP(pa, 0); cond.twindow = pCheckInfo->win; } terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); + pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQuery->tableGroupInfo, pQInfo->qId, &pQuery->memRef); // update the query time window pQuery->window = cond.twindow; - if (pQInfo->tableGroupInfo.numOfTables == 0) { - pQInfo->tableqinfoGroupInfo.numOfTables = 0; + if (pQuery->tableGroupInfo.numOfTables == 0) { + pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; } else { - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); + SArray *group = GET_TABLEGROUP(pRuntimeEnv, i); size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { @@ -4635,26 +3966,24 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) } } } else if (isPointInterpoQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQuery->tableGroupInfo, pQInfo->qId, &pQuery->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQuery->tableGroupInfo, pQInfo->qId, &pQuery->memRef); } return terrno; } -static SFillColInfo* createFillColInfo(SQuery* pQuery) { - int32_t numOfCols = getNumOfFinalResCol(pQuery); +static SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, int64_t* fillVal) { int32_t offset = 0; - SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); + SFillColInfo* pFillCol = calloc(numOfOutput, sizeof(SFillColInfo)); if (pFillCol == NULL) { return NULL; } - // TODO refactor - for(int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExprInfo = (pQuery->pExpr2 == NULL)? &pQuery->pExpr1[i]:&pQuery->pExpr2[i]; + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = &pExpr[i]; pFillCol[i].col.bytes = pExprInfo->bytes; pFillCol[i].col.type = (int8_t)pExprInfo->type; @@ -4662,7 +3991,7 @@ static SFillColInfo* createFillColInfo(SQuery* pQuery) { pFillCol[i].tagIndex = -2; pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query pFillCol[i].functionId = pExprInfo->base.functionId; - pFillCol[i].fillVal.i = pQuery->fillVal[i]; + pFillCol[i].fillVal.i = fillVal[i]; offset += pExprInfo->bytes; } @@ -4670,14 +3999,18 @@ static SFillColInfo* createFillColInfo(SQuery* pQuery) { return pFillCol; } -int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { +int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, SArray* prevResult, void *tsdb, int32_t vgId, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + pQuery->tsdb = tsdb; - pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery); - pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); - pRuntimeEnv->timeWindowInterpo = timeWindowInterpoRequired(pQuery); + pQuery->topBotQuery = isTopBottomQuery(pQuery); + pQuery->hasTagResults = hasTagValOutput(pQuery); + pQuery->timeWindowInterpo = timeWindowInterpoRequired(pQuery); + pQuery->stabledev = isStabledev(pQuery); + + pRuntimeEnv->prevResult = prevResult; setScanLimitationByResultBuffer(pQuery); @@ -4686,15 +4019,31 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } - pQInfo->tsdb = tsdb; - pQInfo->vgId = vgId; + pQuery->tsdb = tsdb; + pQuery->vgId = vgId; + pQuery->stableQuery = isSTableQuery; + pQuery->groupbyColumn = isGroupbyColumn(pQuery->pGroupbyExpr); + pQuery->interBufSize = getOutputInterResultBufSize(pQuery); + + pRuntimeEnv->groupResInfo.totalGroup = (int32_t) (isSTableQuery? GET_NUM_OF_TABLEGROUP(pRuntimeEnv):0); pRuntimeEnv->pQuery = pQuery; pRuntimeEnv->pTsBuf = pTsBuf; pRuntimeEnv->cur.vgroupIndex = -1; - pRuntimeEnv->stableQuery = isSTableQuery; - pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->groupbyColumn = isGroupbyNormalCol(pQuery->pGroupbyExpr); + setResultBufSize(pQuery, &pRuntimeEnv->resultInfo); + + if (onlyQueryTags(pQuery)) { + pRuntimeEnv->resultInfo.capacity = 4096; + pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQuery->pExpr1, pQuery->numOfOutput); + } else if (pQuery->queryBlockDist) { + pRuntimeEnv->pTableScanner = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + } else if (isTsCompQuery(pQuery) || isPointInterpoQuery(pQuery)) { + pRuntimeEnv->pTableScanner = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + } else if (needReverseScan(pQuery)) { + pRuntimeEnv->pTableScanner = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQuery), 1); + } else { + pRuntimeEnv->pTableScanner = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQuery)); + } if (pTsBuf != NULL) { int16_t order = (pQuery->order.order == pRuntimeEnv->pTsBuf->tsOrder) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -4702,103 +4051,26 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } int32_t ps = DEFAULT_PAGE_SIZE; - int32_t rowsize = 0; - getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - int32_t TENMB = 1024*1024*10; - - if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) { - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { - int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyColumn) { // group by columns not tags; - type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); - } else { - type = TSDB_DATA_TYPE_INT; // group id - } - - code = initResultRowInfo(&pRuntimeEnv->windowResInfo, 8, type); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - } else if (pRuntimeEnv->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { - int32_t numOfResultRows = getInitialPageNum(pQInfo); - getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - int16_t type = TSDB_DATA_TYPE_NULL; - if (pRuntimeEnv->groupbyColumn) { - type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); - } else { - type = TSDB_DATA_TYPE_TIMESTAMP; - } + getIntermediateBufInfo(pRuntimeEnv, &ps, &pQuery->intermediateResultRowSize); - code = initResultRowInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, type); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + int32_t TENMB = 1024*1024*10; + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId); + if (code != TSDB_CODE_SUCCESS) { + return code; } // create runtime environment - code = setupQueryRuntimeEnv(pRuntimeEnv, pQuery->order.order); + int32_t numOfTables = (int32_t)pQuery->tableGroupInfo.numOfTables; + pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); + code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQuery->tableGroupInfo.numOfTables); if (code != TSDB_CODE_SUCCESS) { return code; } - if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { - SFillColInfo* pColInfo = createFillColInfo(pQuery); - STimeWindow w = TSWINDOW_INITIALIZER; - - TSKEY sk = MIN(pQuery->window.skey, pQuery->window.ekey); - TSKEY ek = MAX(pQuery->window.skey, pQuery->window.ekey); - getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w); - - int32_t numOfCols = getNumOfFinalResCol(pQuery); - pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, numOfCols, - pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision, - pQuery->fillType, pColInfo, pQInfo); - } - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); return TSDB_CODE_SUCCESS; } -static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { - SQuery *pQuery = pRuntimeEnv->pQuery; - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); - if (pResInfo != NULL) { - pResInfo->complete = false; - } - } -} - -// TODO refactor: setAdditionalInfo -static FORCE_INLINE void setEnvForEachBlock(SQInfo* pQInfo, STableQueryInfo* pTableQueryInfo, SDataBlockInfo* pBlockInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - TSKEY nextKey = pBlockInfo->window.skey; - setIntervalQueryRange(pQInfo, nextKey); - - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { - setAdditionalInfo(pQInfo, pTableQueryInfo->pTable, pTableQueryInfo); - } - } else { // non-interval query - setExecutionContext(pQInfo, pTableQueryInfo->groupIndex, pBlockInfo->window.ekey + step); - } -} - static void doTableQueryInfoTimeWindowCheck(SQuery* pQuery, STableQueryInfo* pTableQueryInfo) { if (QUERY_IS_ASC_QUERY(pQuery)) { assert( @@ -4813,1051 +4085,1299 @@ static void doTableQueryInfoTimeWindowCheck(SQuery* pQuery, STableQueryInfo* pTa } } -static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery* pQuery = pRuntimeEnv->pQuery; - SQueryCostInfo* summary = &pRuntimeEnv->summary; +STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win) { + STsdbQueryCond cond = { + .colList = pQuery->colList, + .order = pQuery->order.order, + .numOfCols = pQuery->numOfCols, + .type = BLOCK_LOAD_OFFSET_SEQ_ORDER, + .loadExternalRows = false, + }; - int64_t st = taosGetTimestampMs(); + TIME_WINDOW_COPY(cond.twindow, *win); + return cond; +} - TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle; - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; +static STableIdInfo createTableIdInfo(STableQueryInfo* pTableQueryInfo) { + STableIdInfo tidInfo; + STableId* id = TSDB_TABLEID(pTableQueryInfo->pTable); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + tidInfo.uid = id->uid; + tidInfo.tid = id->tid; + tidInfo.key = pTableQueryInfo->lastKey; - while (tsdbNextDataBlock(pQueryHandle)) { - summary->totalBlocks += 1; + return tidInfo; +} - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } +static void updateTableIdInfo(STableQueryInfo* pTableQueryInfo, SSDataBlock* pBlock, SHashObj* pTableIdInfo, int32_t order) { + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + pTableQueryInfo->lastKey = ((order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey) + step; - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - STableQueryInfo **pTableQueryInfo = (STableQueryInfo**) taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); - if(pTableQueryInfo == NULL) { - break; - } + STableIdInfo tidInfo = createTableIdInfo(pTableQueryInfo); + STableIdInfo *idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); + if (idinfo != NULL) { + assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); + idinfo->key = tidInfo.key; + } else { + taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); + } +} - pQuery->current = *pTableQueryInfo; - doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); +static void doCloseAllTimeWindow(SQueryRuntimeEnv* pRuntimeEnv) { + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + for (int32_t i = 0; i < numOfGroup; ++i) { + SArray* group = GET_TABLEGROUP(pRuntimeEnv, i); - if (!pRuntimeEnv->groupbyColumn) { - setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); + size_t num = taosArrayGetSize(group); + for (int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(group, j); + closeAllResultRows(&item->resInfo); } + } +} - uint32_t status = 0; - SDataStatis *pStatis = NULL; - SArray *pDataBlock = NULL; - - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); - if (ret != TSDB_CODE_SUCCESS) { - break; +static SSDataBlock* doTableScanImpl(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + + STableScanInfo* pTableScanInfo = pOperator->info; + SSDataBlock* pBlock = &pTableScanInfo->block; + SQuery* pQuery = pOperator->pRuntimeEnv->pQuery; + STableGroupInfo* pTableGroupInfo = &pOperator->pRuntimeEnv->tableqinfoGroupInfo; + + while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { + longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - if (status == BLK_DATA_DISCARD) { - pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; + pTableScanInfo->numOfBlocks += 1; + tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + + // todo opt + if (pTableGroupInfo->numOfTables > 1 || (pQuery->current == NULL && pTableGroupInfo->numOfTables == 1)) { + STableQueryInfo** pTableQueryInfo = + (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.tid, sizeof(pBlock->info.tid)); + if (pTableQueryInfo == NULL) { + break; + } + + pQuery->current = *pTableQueryInfo; + doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); } - summary->totalRows += blockInfo.rows; - stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); + // this function never returns error? + uint32_t status; + int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pOperator->pRuntimeEnv->env, code); + } - qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " - "lastKey:%" PRId64, - pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, - pQuery->current->lastKey); - } + // current block is ignored according to filter result by block statistics data, continue load the next block + if (status == BLK_DATA_DISCARD || pBlock->info.rows == 0) { + continue; + } - if (terrno != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, terrno); + return pBlock; } - updateWindowResNumOfRes(pRuntimeEnv); - - int64_t et = taosGetTimestampMs(); - return et - st; + return NULL; } -static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +static SSDataBlock* doTableScan(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - SArray *group = GET_TABLEGROUP(pQInfo, 0); - STableQueryInfo* pCheckInfo = taosArrayGetP(group, index); + STableScanInfo *pTableScanInfo = pOperator->info; + SQueryRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; + SQuery* pQuery = pRuntimeEnv->pQuery; - if (pRuntimeEnv->hasTagResults || pRuntimeEnv->pTsBuf != NULL) { - setTagVal(pRuntimeEnv, pCheckInfo->pTable, pQInfo->tsdb); - } + SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; - STableId* id = TSDB_TABLEID(pCheckInfo->pTable); - qDebug("QInfo:%p query on (%d): uid:%" PRIu64 ", tid:%d, qrange:%" PRId64 "-%" PRId64, pQInfo, index, - id->uid, id->tid, pCheckInfo->lastKey, pCheckInfo->win.ekey); + while (pTableScanInfo->current < pTableScanInfo->times) { + SSDataBlock* p = doTableScanImpl(pOperator); + if (p != NULL) { + return p; + } - STsdbQueryCond cond = { - .twindow = {pCheckInfo->lastKey, pCheckInfo->win.ekey}, - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; + if (++pTableScanInfo->current >= pTableScanInfo->times) { + if (pTableScanInfo->reverseTimes <= 0) { + return NULL; + } else { + break; + } + } - // todo refactor - SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayInit(1, sizeof(STableKeyInfo)); + // do prepare for the next round table scan operation + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); - STableKeyInfo info = {.pTable = pCheckInfo->pTable, .lastKey = pCheckInfo->lastKey}; - taosArrayPush(tx, &info); + setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); + pRuntimeEnv->scanFlag = REPEAT_SCAN; - taosArrayPush(g1, &tx); - STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; + if (pRuntimeEnv->pTsBuf) { + bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); + assert(ret); + } - // include only current table - if (pRuntimeEnv->pQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; - } + if (pResultRowInfo->size > 0) { + pResultRowInfo->curIndex = 0; + pResultRowInfo->prevSKey = pResultRowInfo->pResult[0]->win.skey; + } - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); - taosArrayDestroy(tx); - taosArrayDestroy(g1); - if (pRuntimeEnv->pQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); + qDebug("QInfo:%"PRIu64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); } - if (pRuntimeEnv->pTsBuf != NULL) { - tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; - - if (pRuntimeEnv->cur.vgroupIndex == -1) { - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, pTag); - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); - } else { - qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64); - } + if (pTableScanInfo->reverseTimes > 0) { + setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); - return false; - } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, - cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, - cur.blockIndex, cur.tsIndex); - } - } - } else { - STSElem elem = tsBufGetElem(pRuntimeEnv->pTsBuf); - if (tVariantCompare(elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { + qDebug("QInfo:%"PRIu64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTsBuf, pQInfo->vgId, pTag); - // failed to find data with the specified tag value and vnodeId - if (!tsBufIsValidElem(&elem1)) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); - } else { - qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64); - } + pRuntimeEnv->scanFlag = REVERSE_SCAN; - return false; - } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, cur.blockIndex, cur.tsIndex); - } - } + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; + pTableScanInfo->order = cond.order; - } else { - tsBufSetCursor(pRuntimeEnv->pTsBuf, &pRuntimeEnv->cur); - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64, cur.blockIndex, cur.tsIndex); - } - } + if (pResultRowInfo->size > 0) { + pResultRowInfo->curIndex = pResultRowInfo->size-1; + pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->size-1]->win.skey; + } + + SSDataBlock* p = doTableScanImpl(pOperator); + if (p != NULL) { + return p; } } - initCtxOutputBuf(pRuntimeEnv); - return true; + return NULL; } -STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win) { - STsdbQueryCond cond = { - .colList = pQuery->colList, - .order = pQuery->order.order, - .numOfCols = pQuery->numOfCols, - }; +static SSDataBlock* doBlockInfoScan(void* param) { + SOperatorInfo *pOperator = (SOperatorInfo*)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - TIME_WINDOW_COPY(cond.twindow, *win); - return cond; -} + STableScanInfo *pTableScanInfo = pOperator->info; -static STableIdInfo createTableIdInfo(SQuery* pQuery) { - assert(pQuery != NULL && pQuery->current != NULL); + STableBlockDist tableBlockDist = {0}; + tableBlockDist.numOfTables = (int32_t)pOperator->pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + tableBlockDist.dataBlockInfos = taosArrayInit(512, sizeof(SFileBlockInfo)); - STableIdInfo tidInfo; - STableId* id = TSDB_TABLEID(pQuery->current->pTable); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); - tidInfo.uid = id->uid; - tidInfo.tid = id->tid; - tidInfo.key = pQuery->current->lastKey; + SSDataBlock* pBlock = &pTableScanInfo->block; + pBlock->info.rows = 1; + pBlock->info.numOfCols = 1; - return tidInfo; -} + SBufferWriter bw = tbufInitWriter(NULL, false); + blockDistInfoToBinary(&tableBlockDist, &bw); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); -static void updateTableIdInfo(SQuery* pQuery, SHashObj* pTableIdInfo) { - STableIdInfo tidInfo = createTableIdInfo(pQuery); - STableIdInfo* idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); - if (idinfo != NULL) { - assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); - idinfo->key = tidInfo.key; - } else { - taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); - } -} + int32_t len = (int32_t) tbufTell(&bw); + pColInfo->pData = malloc(len + sizeof(int32_t)); -/** - * super table query handler - * 1. super table projection query, group-by on normal columns query, ts-comp query - * 2. point interpolation query, last row query - * - * @param pQInfo - */ -static void sequentialTableProcess(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - setQueryStatus(pQuery, QUERY_COMPLETED); + *(int32_t*) pColInfo->pData = len; + memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len); - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + tbufCloseWriter(&bw); - if (isPointInterpoQuery(pQuery)) { - resetDefaultResInfoOutputBuf(pRuntimeEnv); - assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); + SArray* g = GET_TABLEGROUP(pOperator->pRuntimeEnv, 0); + pOperator->pRuntimeEnv->pQuery->current = taosArrayGetP(g, 0); - while (pQInfo->groupIndex < numOfGroups) { - SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); + pOperator->status = OP_EXEC_DONE; + return pBlock; +} - qDebug("QInfo:%p point interpolation query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, - pQInfo->groupIndex, numOfGroups, group); - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime) { + assert(repeatTime > 0); - SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayClone(group); - taosArrayPush(g1, &tx); + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQuery->order.order; + pInfo->current = 0; - STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableScanOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQuery->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScan; - // include only current table - if (pRuntimeEnv->pQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; - } + return pOperator; +} - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); +SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - taosArrayDestroy(tx); - taosArrayDestroy(g1); - if (pRuntimeEnv->pQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = 1; + pInfo->reverseTimes = 0; + pInfo->order = pRuntimeEnv->pQuery->order.order; + pInfo->current = 0; - initCtxOutputBuf(pRuntimeEnv); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableSeqScanOperator"; + pOperator->operatorType = OP_TableSeqScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = pRuntimeEnv->pQuery->numOfCols; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doTableScanImpl; - SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); - assert(taosArrayGetSize(s) >= 1); + return pOperator; +} - setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); - taosArrayDestroy(s); +SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - // here we simply set the first table as current table - SArray *first = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); - pQuery->current = taosArrayGetP(first, 0); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); + SColumnInfoData infoData = {{0}}; + infoData.info.type = TSDB_DATA_TYPE_BINARY; + infoData.info.bytes = 1024; + infoData.info.colId = TSDB_BLOCK_DIST_COLUMN_INDEX; + taosArrayPush(pInfo->block.pDataBlock, &infoData); - int64_t numOfRes = getNumOfResult(pRuntimeEnv); - if (numOfRes > 0) { - pQuery->rec.rows += numOfRes; - forwardCtxOutputBuf(pRuntimeEnv, numOfRes); - } + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableBlockInfoScanOperator"; + pOperator->operatorType = OP_TableBlockInfoScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->numOfOutput = pRuntimeEnv->pQuery->numOfCols; + pOperator->exec = doBlockInfoScan; - skipResults(pRuntimeEnv); - pQInfo->groupIndex += 1; + return pOperator; +} - // enable execution for next table, when handling the projection query - enableExecutionForNextTable(pRuntimeEnv); +void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { + assert(pTableScanInfo != NULL && pDownstream != NULL); - if (pQuery->rec.rows >= pQuery->rec.capacity) { - setQueryStatus(pQuery, QUERY_RESBUF_FULL); - break; - } - } - } else if (pRuntimeEnv->groupbyColumn) { // group-by on normal columns query - while (pQInfo->groupIndex < numOfGroups) { - SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); + pTableScanInfo->pExpr = pDownstream->pExpr; // TODO refactor to use colId instead of pExpr + pTableScanInfo->numOfOutput = pDownstream->numOfOutput; - qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, - numOfGroups); + if (pDownstream->operatorType == OP_Aggregate || pDownstream->operatorType == OP_MultiTableAggregate) { + SAggOperatorInfo* pAggInfo = pDownstream->info; - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + pTableScanInfo->pCtx = pAggInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pAggInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_TimeWindow) { + STableIntervalOperatorInfo *pIntervalInfo = pDownstream->info; - SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayClone(group); - taosArrayPush(g1, &tx); + pTableScanInfo->pCtx = pIntervalInfo->pCtx; + pTableScanInfo->pResultRowInfo = &pIntervalInfo->resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pIntervalInfo->rowCellInfoOffset; - STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; + } else if (pDownstream->operatorType == OP_Groupby) { + SGroupbyOperatorInfo *pGroupbyInfo = pDownstream->info; - // include only current table - if (pRuntimeEnv->pQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; - } + pTableScanInfo->pCtx = pGroupbyInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pGroupbyInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pGroupbyInfo->binfo.rowCellInfoOffset; - // no need to update the lastkey for each table - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo, &pQInfo->memRef); + } else if (pDownstream->operatorType == OP_MultiTableTimeInterval) { + STableIntervalOperatorInfo *pInfo = pDownstream->info; - taosArrayDestroy(g1); - taosArrayDestroy(tx); - if (pRuntimeEnv->pQueryHandle == NULL) { - longjmp(pRuntimeEnv->env, terrno); - } + pTableScanInfo->pCtx = pInfo->pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; + + } else if (pDownstream->operatorType == OP_Arithmetic) { + SArithOperatorInfo *pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else if (pDownstream->operatorType == OP_SessionWindow) { + SSWindowOperatorInfo* pInfo = pDownstream->info; + + pTableScanInfo->pCtx = pInfo->binfo.pCtx; + pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; + pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset; + } else { + assert(0); + } +} - SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); - assert(taosArrayGetSize(s) >= 1); +static SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { + assert(repeatTime > 0); - setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); + STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); + pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->current = 0; + pInfo->order = pRuntimeEnv->pQuery->order.order; - // here we simply set the first table as current table - scanMultiTableDataBlocks(pQInfo); - pQInfo->groupIndex += 1; + SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); + pOptr->name = "DataBlocksOptimizedScanOperator"; + pOptr->operatorType = OP_DataBlocksOptScan; + pOptr->pRuntimeEnv = pRuntimeEnv; + pOptr->blockingOptr = false; + pOptr->info = pInfo; + pOptr->exec = doTableScan; - taosArrayDestroy(s); + return pOptr; +} - // no results generated for current group, continue to try the next group - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pWindowResInfo->size <= 0) { - continue; - } +static int32_t getTableScanOrder(STableScanInfo* pTableScanInfo) { + return pTableScanInfo->order; +} - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns +// this is a blocking operator +static SSDataBlock* doAggregate(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - SResultRow *pResult = pWindowResInfo->pResult[i]; - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultRowCellInfo *pCell = getResultCell(pRuntimeEnv, pResult, j); - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); - } - } + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; - qDebug("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size, - pQInfo->groupIndex); - int32_t currentGroupIndex = pQInfo->groupIndex; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - pQuery->rec.rows = 0; - pQInfo->groupIndex = 0; + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; - ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size); - copyFromWindowResToSData(pQInfo, pWindowResInfo); + SOperatorInfo* upstream = pOperator->upstream; - pQInfo->groupIndex = currentGroupIndex; // restore the group index - assert(pQuery->rec.rows == pWindowResInfo->size); - resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { break; } - } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTsBuf == NULL && !isTSCompQuery(pQuery)) { - //super table projection query with identical query time range for all tables. - SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - resetDefaultResInfoOutputBuf(pRuntimeEnv); - - SArray *group = GET_TABLEGROUP(pQInfo, 0); - assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && - 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); - - void *pQueryHandle = pRuntimeEnv->pQueryHandle; - if (pQueryHandle == NULL) { - STsdbQueryCond con = createTsdbQueryCond(pQuery, &pQuery->window); - pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &con, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - pQueryHandle = pRuntimeEnv->pQueryHandle; - } - // skip blocks without load the actual data block from file if no filter condition present - // skipBlocks(&pQInfo->runtimeEnv); - // if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { - // setQueryStatus(pQuery, QUERY_COMPLETED); - // return; - // } + setTagValue(pOperator, pQuery->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - if (pQuery->prjInfo.vgroupLimit != -1) { - assert(pQuery->limit.limit == -1 && pQuery->limit.offset == 0); - } else if (pQuery->limit.limit != -1) { - assert(pQuery->prjInfo.vgroupLimit == -1); + if (upstream->operatorType == OP_DataBlocksOptScan) { + STableScanInfo* pScanInfo = upstream->info; + order = getTableScanOrder(pScanInfo); } - bool hasMoreBlock = true; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - SQueryCostInfo *summary = &pRuntimeEnv->summary; - while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) { - summary->totalBlocks += 1; - - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } - - tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - STableQueryInfo **pTableQueryInfo = - (STableQueryInfo **) taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); - if (pTableQueryInfo == NULL) { - break; - } - - pQuery->current = *pTableQueryInfo; - doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); - - if (pRuntimeEnv->hasTagResults) { - setTagVal(pRuntimeEnv, pQuery->current->pTable, pQInfo->tsdb); - } - - if (pQuery->prjInfo.vgroupLimit > 0 && pQuery->current->windowResInfo.size > pQuery->prjInfo.vgroupLimit) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } - - // it is a super table ordered projection query, check for the number of output for each vgroup - if (pQuery->prjInfo.vgroupLimit > 0 && pQuery->rec.rows >= pQuery->prjInfo.vgroupLimit) { - if (QUERY_IS_ASC_QUERY(pQuery) && blockInfo.window.skey >= pQuery->prjInfo.ts) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } else if (!QUERY_IS_ASC_QUERY(pQuery) && blockInfo.window.ekey <= pQuery->prjInfo.ts) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } - } + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + doAggregateImpl(pOperator, pQuery->window.skey, pInfo->pCtx, pBlock); + } - uint32_t status = 0; - SDataStatis *pStatis = NULL; - SArray *pDataBlock = NULL; + pOperator->status = OP_EXEC_DONE; + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); - int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo, - &pStatis, &pDataBlock, &status); - if (ret != TSDB_CODE_SUCCESS) { - break; - } + finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); + pInfo->pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); - if(status == BLK_DATA_DISCARD) { - pQuery->current->lastKey = - QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; - continue; - } + return pInfo->pRes; +} - ensureOutputBuffer(pRuntimeEnv, &blockInfo); - int64_t prev = getNumOfResult(pRuntimeEnv); +static SSDataBlock* doSTableAggregate(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; - int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); + SAggOperatorInfo* pAggInfo = pOperator->info; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; - summary->totalRows += blockInfo.rows; - qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, - GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, - pQuery->current->lastKey); + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - pQuery->rec.rows = getNumOfResult(pRuntimeEnv); + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); - int64_t inc = pQuery->rec.rows - prev; - pQuery->current->windowResInfo.size += (int32_t) inc; + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } - // the flag may be set by tableApplyFunctionsOnBlock, clear it here - CLEAR_QUERY_STATUS(pQuery, QUERY_COMPLETED); + return pInfo->pRes; + } - updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; - if (pQuery->prjInfo.vgroupLimit >= 0) { - if (((pQuery->rec.rows + pQuery->rec.total) < pQuery->prjInfo.vgroupLimit) || ((pQuery->rec.rows + pQuery->rec.total) > pQuery->prjInfo.vgroupLimit && prev < pQuery->prjInfo.vgroupLimit)) { - if (QUERY_IS_ASC_QUERY(pQuery) && pQuery->prjInfo.ts < blockInfo.window.ekey) { - pQuery->prjInfo.ts = blockInfo.window.ekey; - } else if (!QUERY_IS_ASC_QUERY(pQuery) && pQuery->prjInfo.ts > blockInfo.window.skey) { - pQuery->prjInfo.ts = blockInfo.window.skey; - } - } - } else { - // the limitation of output result is reached, set the query completed - skipResults(pRuntimeEnv); - if (limitResults(pRuntimeEnv)) { - setQueryStatus(pQuery, QUERY_COMPLETED); - SET_STABLE_QUERY_OVER(pQInfo); - break; - } - } + SOperatorInfo* upstream = pOperator->upstream; - // while the output buffer is full or limit/offset is applied, query may be paused here - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL|QUERY_COMPLETED)) { - break; - } + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; } - if (!hasMoreBlock) { - setQueryStatus(pQuery, QUERY_COMPLETED); - SET_STABLE_QUERY_OVER(pQInfo); - } - } else { - /* - * the following two cases handled here. - * 1. ts-comp query, and 2. the super table projection query with different query time range for each table. - * If the subgroup index is larger than 0, results generated by group by tbname,k is existed. - * we need to return it to client in the first place. - */ - if (pQInfo->groupIndex > 0) { - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - pQuery->rec.total += pQuery->rec.rows; + setTagValue(pOperator, pRuntimeEnv->pQuery->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - if (pQuery->rec.rows > 0) { - return; - } + if (upstream->operatorType == OP_DataBlocksOptScan) { + STableScanInfo* pScanInfo = upstream->info; + order = getTableScanOrder(pScanInfo); } - // all data have returned already - if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) { - return; - } + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - resetDefaultResInfoOutputBuf(pRuntimeEnv); - resetResultRowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? pBlock->info.window.ekey + 1:pBlock->info.window.skey-1; + setExecutionContext(pRuntimeEnv, pInfo, pOperator->numOfOutput, pQuery->current->groupIndex, key); + doAggregateImpl(pOperator, pQuery->window.skey, pInfo->pCtx, pBlock); + } - SArray *group = GET_TABLEGROUP(pQInfo, 0); - assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && - 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->resultRowInfo); - while (pQInfo->tableIndex < pQInfo->tableqinfoGroupInfo.numOfTables) { - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); - } + updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, + pInfo->rowCellInfoOffset); - pQuery->current = taosArrayGetP(group, pQInfo->tableIndex); - if (!multiTableMultioutputHelper(pQInfo, pQInfo->tableIndex)) { - pQInfo->tableIndex++; - continue; - } + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->resultRowInfo); - // TODO handle the limit offset problem - if (pQuery->numOfFilterCols == 0 && pQuery->limit.offset > 0) { - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - pQInfo->tableIndex++; - continue; - } - } + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + } - scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - skipResults(pRuntimeEnv); + return pInfo->pRes; +} - // the limitation of output result is reached, set the query completed - if (limitResults(pRuntimeEnv)) { - SET_STABLE_QUERY_OVER(pQInfo); - break; - } +static SSDataBlock* doArithmeticOperation(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; - // enable execution for next table, when handling the projection query - enableExecutionForNextTable(pRuntimeEnv); - - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - /* - * query range is identical in terms of all meters involved in query, - * so we need to restore them at the *beginning* of query on each meter, - * not the consecutive query on meter on which is aborted due to buffer limitation - * to ensure that, we can reset the query range once query on a meter is completed. - */ - pQInfo->tableIndex++; - updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); - - // if the buffer is full or group by each table, we need to jump out of the loop - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - break; - } + SArithOperatorInfo* pArithInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + SOptrBasicInfo *pInfo = &pArithInfo->binfo; - if (pRuntimeEnv->pTsBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTsBuf->cur; - } + SSDataBlock* pRes = pInfo->pRes; + int32_t order = pRuntimeEnv->pQuery->order.order; - } else { - // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter - if (pQuery->rec.rows == 0) { - assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); - continue; - } else { - // buffer is full, wait for the next round to retrieve data from current meter - assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); - break; - } - } - } + pRes->info.rows = 0; - if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) { - setQueryStatus(pQuery, QUERY_COMPLETED); + while(1) { + SSDataBlock* pBlock = pOperator->upstream->exec(pOperator->upstream); + if (pBlock == NULL) { + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + break; } - /* - * 1. super table projection query, group-by on normal columns query, ts-comp query - * 2. point interpolation query, last row query - * - * group-by on normal columns query and last_row query do NOT invoke the finalizer here, - * since the finalize stage will be done at the client side. - * - * projection query, point interpolation query do not need the finalizer. - * - * Only the ts-comp query requires the finalizer function to be executed here. - */ - if (isTSCompQuery(pQuery)) { - finalizeQueryResult(pRuntimeEnv); - } + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->pQuery->current; - if (pRuntimeEnv->pTsBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTsBuf->cur; - } + // todo dynamic set tags + setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); - qDebug("QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 - " points returned, total:%" PRId64 ", offset:%" PRId64, - pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, - pQuery->rec.total, pQuery->limit.offset); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); + updateOutputBuf(pArithInfo, pBlock->info.rows); + + arithmeticApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); + + pRes->info.rows = getNumOfResult(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + if (pRes->info.rows >= pRuntimeEnv->resultInfo.threshold) { + break; + } } + + clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); + return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; } -static int32_t doSaveContext(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +static SSDataBlock* doLimit(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); + SLimitOperatorInfo* pInfo = pOperator->info; - if (pRuntimeEnv->pTsBuf != NULL) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); + SSDataBlock* pBlock = pOperator->upstream->exec(pOperator->upstream); + if (pBlock == NULL) { + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + pOperator->status = OP_EXEC_DONE; + return NULL; } - STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); + if (pInfo->total + pBlock->info.rows >= pInfo->limit) { + pBlock->info.rows = (int32_t) (pInfo->limit - pInfo->total); - // clean unused handle - if (pRuntimeEnv->pSecQueryHandle != NULL) { - tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); - } + pInfo->total = pInfo->limit; - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - switchCtxOrder(pRuntimeEnv); - disableFuncInReverseScan(pQInfo); - setupQueryRangeForReverseScan(pQInfo); + setQueryStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + pOperator->status = OP_EXEC_DONE; + } else { + pInfo->total += pBlock->info.rows; + } - pRuntimeEnv->prevGroupId = INT32_MIN; - pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); - return (pRuntimeEnv->pSecQueryHandle == NULL)? -1:0; + return pBlock; } -static void doRestoreContext(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +// TODO add log +static SSDataBlock* doOffset(void* param) { + SOperatorInfo *pOperator = (SOperatorInfo *)param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - SWITCH_ORDER(pQuery->order.order); + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pRuntimeEnv->pTsBuf != NULL) { - SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); - } + while (1) { + SSDataBlock *pBlock = pOperator->upstream->exec(pOperator->upstream); + if (pBlock == NULL) { + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + pOperator->status = OP_EXEC_DONE; + return NULL; + } - switchCtxOrder(pRuntimeEnv); - SET_MASTER_SCAN_FLAG(pRuntimeEnv); -} + if (pRuntimeEnv->currentOffset == 0) { + return pBlock; + } else if (pRuntimeEnv->currentOffset >= pBlock->info.rows) { + pRuntimeEnv->currentOffset -= pBlock->info.rows; + } else { + int32_t remain = (int32_t)(pBlock->info.rows - pRuntimeEnv->currentOffset); + pBlock->info.rows = remain; -static void doCloseAllTimeWindowAfterScan(SQInfo *pQInfo) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData *pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); - for (int32_t i = 0; i < numOfGroup; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); - - size_t num = taosArrayGetSize(group); - for (int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(group, j); - closeAllResultRows(&item->windowResInfo); + int16_t bytes = pColInfoData->info.bytes; + memmove(pColInfoData->pData, pColInfoData->pData + bytes * pRuntimeEnv->currentOffset, remain * bytes); } + + pRuntimeEnv->currentOffset = 0; + return pBlock; } - } else { // close results for group result - closeAllResultRows(&pQInfo->runtimeEnv.windowResInfo); } } -static void multiTableQueryProcess(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; +static SSDataBlock* doIntervalAgg(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - if (pQInfo->groupIndex > 0) { - /* - * if the groupIndex > 0, the query process must be completed yet, we only need to - * copy the data into output buffer - */ - if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - copyResToQueryResultBuf(pQInfo, pQuery); - } else { - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - qDebug("QInfo:%p current:%"PRId64", total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total); - return; + return pIntervalInfo->pRes; } - qDebug("QInfo:%p query start, qrange:%" PRId64 "-%" PRId64 ", order:%d, forward scan start", pQInfo, - pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; + STimeWindow win = pQuery->window; + + SOperatorInfo* upstream = pOperator->upstream; - // do check all qualified data blocks - int64_t el = scanMultiTableDataBlocks(pQInfo); - qDebug("QInfo:%p master scan completed, elapsed time: %" PRId64 "ms, reverse scan start", pQInfo, el); + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; + } - // query error occurred or query is killed, abort current execution - if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { - qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQuery->order.order); + hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); } - // close all time window results - doCloseAllTimeWindowAfterScan(pQInfo); + // restore the value + pQuery->order.order = order; + pQuery->window = win; - if (needReverseScan(pQuery)) { - int32_t code = doSaveContext(pQInfo); - if (code == TSDB_CODE_SUCCESS) { - el = scanMultiTableDataBlocks(pQInfo); - qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); - doRestoreContext(pQInfo); - } else { - pQInfo->code = code; - } - } else { - qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); - } + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pIntervalInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); - setQueryStatus(pQuery, QUERY_COMPLETED); + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pQInfo->code != TSDB_CODE_SUCCESS || isQueryKilled(pQInfo)) { - qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); - //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) { - int32_t code = mergeGroupResult(pQInfo); - if (code == TSDB_CODE_SUCCESS) { - copyResToQueryResultBuf(pQInfo, pQuery); + return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; +} -#ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); -#endif - } else { // set the error code - pQInfo->code = code; +static SSDataBlock* doSTableIntervalAgg(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + + if (pOperator->status == OP_RES_TO_RETURN) { + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - } else { // not a interval query - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + + return pIntervalInfo->pRes; } - // handle the limitation of output buffer - qDebug("QInfo:%p points returned:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); -} + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; -static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) { - SArithmeticSupport *pSupport = (SArithmeticSupport *) param; - SExprInfo* pExprInfo = (SExprInfo*) pSupport->exprList; + SOperatorInfo* upstream = pOperator->upstream; - int32_t index = -1; - for (int32_t i = 0; i < pSupport->numOfCols; ++i) { - if (colId == pExprInfo[i].base.resColId) { - index = i; + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { break; } - } - assert(index >= 0 && index < pSupport->numOfCols); - return pSupport->data[index] + pSupport->offset * pExprInfo[index].bytes; -} + // the pDataBlock are always the same one, no need to call this again + STableQueryInfo* pTableQueryInfo = pRuntimeEnv->pQuery->current; -static void doSecondaryArithmeticProcess(SQuery* pQuery) { - if (pQuery->numOfExpr2 == 0) { - return; + setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); + setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQuery->order.order); + setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); + + hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); } - SArithmeticSupport arithSup = {0}; - tFilePage **data = calloc(pQuery->numOfExpr2, POINTER_BYTES); - for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - int32_t bytes = pQuery->pExpr2[i].bytes; - data[i] = (tFilePage *)malloc((size_t)(bytes * pQuery->rec.rows) + sizeof(tFilePage)); + pOperator->status = OP_RES_TO_RETURN; + pQuery->order.order = order; // TODO : restore the order + doCloseAllTimeWindow(pRuntimeEnv); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); + if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - arithSup.offset = 0; - arithSup.numOfCols = (int32_t)pQuery->numOfOutput; - arithSup.exprList = pQuery->pExpr1; - arithSup.data = calloc(arithSup.numOfCols, POINTER_BYTES); + return pIntervalInfo->pRes; +} - for (int32_t k = 0; k < arithSup.numOfCols; ++k) { - arithSup.data[k] = pQuery->sdata[k]->data; +static SSDataBlock* doSessionWindowAgg(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; } - for (int i = 0; i < pQuery->numOfExpr2; ++i) { - SExprInfo *pExpr = &pQuery->pExpr2[i]; + SSWindowOperatorInfo* pWindowInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pWindowInfo->binfo; - // calculate the result from several other columns - SSqlFuncMsg* pSqlFunc = &pExpr->base; - if (pSqlFunc->functionId != TSDB_FUNC_ARITHM) { + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - if (pSqlFunc->functionId == pQuery->pExpr1[j].base.functionId && - pSqlFunc->colInfo.colId == pQuery->pExpr1[j].base.colInfo.colId) { - memcpy(data[i]->data, pQuery->sdata[j]->data, (size_t)(pQuery->pExpr1[j].bytes * pQuery->rec.rows)); - break; - } - } - } else { - arithSup.pArithExpr = pExpr; - arithmeticTreeTraverse(arithSup.pArithExpr->pExpr, (int32_t)pQuery->rec.rows, data[i]->data, &arithSup, TSDB_ORDER_ASC, - getArithemicInputSrc); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - } - for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - memcpy(pQuery->sdata[i]->data, data[i]->data, (size_t)(pQuery->pExpr2[i].bytes * pQuery->rec.rows)); + return pBInfo->pRes; } - for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - tfree(data[i]); - } + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t order = pQuery->order.order; + STimeWindow win = pQuery->window; - tfree(data); - tfree(arithSup.data); -} + SOperatorInfo* upstream = pOperator->upstream; -/* - * in each query, this function will be called only once, no retry for further result. - * - * select count(*)/top(field,k)/avg(field name) from table_name [where ts>now-1a]; - * select count(*) from table_name group by status_column; - */ -static void tableAggregationProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { + break; + } - SQuery *pQuery = pRuntimeEnv->pQuery; - if (!pRuntimeEnv->topBotQuery && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore. - return; + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQuery->order.order); + doSessionWindowAggImpl(pOperator, pWindowInfo, pBlock); } - scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); - finalizeQueryResult(pRuntimeEnv); + // restore the value + pQuery->order.order = order; + pQuery->window = win; + + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pBInfo->resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - // since the numOfRows must be identical for all sql functions that are allowed to be executed simutaneously. - pQuery->rec.rows = getNumOfResult(pRuntimeEnv); - doSecondaryArithmeticProcess(pQuery); + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); - if (isQueryKilled(pQInfo)) { - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - skipResults(pRuntimeEnv); - limitResults(pRuntimeEnv); + return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes; } -static void tableProjectionProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - - // for ts_comp query, re-initialized is not allowed - SQuery *pQuery = pRuntimeEnv->pQuery; - if (!isTSCompQuery(pQuery)) { - resetDefaultResInfoOutputBuf(pRuntimeEnv); +static SSDataBlock* hashGroupbyAggregate(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; } - // skip blocks without load the actual data block from file if no filter condition present - skipBlocks(&pQInfo->runtimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { - setQueryStatus(pQuery, QUERY_COMPLETED); - return; - } + SGroupbyOperatorInfo *pInfo = pOperator->info; - while (1) { - scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - finalizeQueryResult(pRuntimeEnv); + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + if (pOperator->status == OP_RES_TO_RETURN) { + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - pQuery->rec.rows = getNumOfResult(pRuntimeEnv); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols > 0 && pQuery->rec.rows > 0) { - skipResults(pRuntimeEnv); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } - /* - * 1. if pQuery->size == 0, pQuery->limit.offset >= 0, still need to check data - * 2. if pQuery->size > 0, pQuery->limit.offset must be 0 - */ - if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { + return pInfo->binfo.pRes; + } + + SOperatorInfo* upstream = pOperator->upstream; + + while(1) { + SSDataBlock* pBlock = upstream->exec(upstream); + if (pBlock == NULL) { break; } - qDebug("QInfo:%p skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, - pQInfo, pQuery->limit.offset, pQuery->current->lastKey, pQuery->current->win.ekey); + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQuery->order.order); + setTagValue(pOperator, pRuntimeEnv->pQuery->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); + if (pInfo->colIndex == -1) { + pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQuery->pGroupbyExpr, pBlock); + } - resetDefaultResInfoOutputBuf(pRuntimeEnv); + doHashGroupbyAgg(pOperator, pInfo, pBlock); } - limitResults(pRuntimeEnv); - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - qDebug("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, - pQuery->current->lastKey, pQuery->window.ekey); - } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - STableIdInfo tidInfo = createTableIdInfo(pQuery); - taosHashPut(pQInfo->arrTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); + pOperator->status = OP_RES_TO_RETURN; + closeAllResultRows(&pInfo->binfo.resultRowInfo); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + + if (!pRuntimeEnv->pQuery->stableQuery) { // finalize include the update of result rows + finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + } else { + updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); } - if (!isTSCompQuery(pQuery)) { - assert(pQuery->rec.rows <= pQuery->rec.capacity); + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); + toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); + + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; } + + return pInfo->binfo.pRes; } -// handle time interval query on table -static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv); - SQuery *pQuery = pRuntimeEnv->pQuery; +static SSDataBlock* doFill(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - TSKEY newStartKey = QUERY_IS_ASC_QUERY(pQuery)? INT64_MIN:INT64_MAX; + SFillOperatorInfo *pInfo = pOperator->info; + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - // skip blocks without load the actual data block from file if no filter condition present - if (!pRuntimeEnv->groupbyColumn) { - skipTimeInterval(pRuntimeEnv, &newStartKey); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { - setQueryStatus(pQuery, QUERY_COMPLETED); - return; - } + if (taosFillHasMoreResults(pInfo->pFillInfo)) { + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, (int32_t)pRuntimeEnv->resultInfo.capacity); + return pInfo->pRes; } - scanOneTableDataBlocks(pRuntimeEnv, newStartKey); - assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED)); + while(1) { + SSDataBlock *pBlock = pOperator->upstream->exec(pOperator->upstream); + if (pBlock == NULL) { + if (pInfo->totalInputRows == 0) { + pOperator->status = OP_EXEC_DONE; + return NULL; + } - finalizeQueryResult(pRuntimeEnv); + taosFillSetStartInfo(pInfo->pFillInfo, 0, pRuntimeEnv->pQuery->window.ekey); + } else { + pInfo->totalInputRows += pBlock->info.rows; - // skip offset result rows - pQuery->rec.rows = 0; + int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQuery->window.ekey:pBlock->info.window.ekey; - // not fill or no result generated during this query - if (pQuery->fillType == TSDB_FILL_NONE || pRuntimeEnv->windowResInfo.size == 0) { - // all data scanned, the group by normal column can return - int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo); - if (pQuery->limit.offset > numOfClosed) { - return; + taosFillSetStartInfo(pInfo->pFillInfo, pBlock->info.rows, ekey); + taosFillSetInputDataBlock(pInfo->pFillInfo, pBlock); + } + + doFillTimeIntervalGapsInResults(pInfo->pFillInfo, pInfo->pRes, pRuntimeEnv->resultInfo.capacity); + return (pInfo->pRes->info.rows > 0)? pInfo->pRes:NULL; + } +} + +// todo set the attribute of query scan count +static int32_t getNumOfScanTimes(SQuery* pQuery) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pExpr1[i].base.functionId; + if (functionId == TSDB_FUNC_STDDEV || functionId == TSDB_FUNC_PERCT) { + return 2; } + } - pQInfo->groupIndex = (int32_t) pQuery->limit.offset; + return 1; +} - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - doSecondaryArithmeticProcess(pQuery); +static void destroyOperatorInfo(SOperatorInfo* pOperator) { + if (pOperator == NULL) { + return; + } - limitResults(pRuntimeEnv); - } else { + if (pOperator->cleanup != NULL) { + pOperator->cleanup(pOperator->info, pOperator->numOfOutput); + } - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - doSecondaryArithmeticProcess(pQuery); + destroyOperatorInfo(pOperator->upstream); + tfree(pOperator->info); + tfree(pOperator); +} + +static SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); - taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage **)pQuery->sdata); + SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t numOfRows = (int32_t)(GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pQuery->topBotQuery, pQuery->stableQuery)); - int32_t numOfFilled = 0; - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - limitResults(pRuntimeEnv); - } + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->seed = rand(); + setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "TableAggregate"; + pOperator->operatorType = OP_Aggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doAggregate; + pOperator->cleanup = destroyBasicOperatorInfo; + return pOperator; +} + +static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { + assert(pInfo != NULL); + + destroySQLFunctionCtx(pInfo->pCtx, numOfOutput); + tfree(pInfo->rowCellInfoOffset); + + cleanupResultRowInfo(&pInfo->resultRowInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { + SOptrBasicInfo* pInfo = (SOptrBasicInfo*) param; + doDestroyBasicInfo(pInfo, numOfOutput); +} + +static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { + SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; + pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); + tfree(pInfo->prevData); +} + +static void destroyArithOperatorInfo(void* param, int32_t numOfOutput) { + SArithOperatorInfo* pInfo = (SArithOperatorInfo*) param; + doDestroyBasicInfo(&pInfo->binfo, numOfOutput); +} + +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { + STagScanInfo* pInfo = (STagScanInfo*) param; + pInfo->pRes = destroyOutputBuf(pInfo->pRes); +} + +SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + + size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup); + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiTableAggregate"; + pOperator->operatorType = OP_MultiTableAggregate; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doSTableAggregate; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SArithOperatorInfo* pInfo = calloc(1, sizeof(SArithOperatorInfo)); + + pInfo->seed = rand(); + pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; + + SOptrBasicInfo* pBInfo = &pInfo->binfo; + pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); + pBInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); + + initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "ArithmeticOperator"; + pOperator->operatorType = OP_Arithmetic; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doArithmeticOperation; + pOperator->cleanup = destroyArithOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { + SLimitOperatorInfo* pInfo = calloc(1, sizeof(SLimitOperatorInfo)); + pInfo->limit = pRuntimeEnv->pQuery->limit.limit; + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "LimitOperator"; + pOperator->operatorType = OP_Limit; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->exec = doLimit; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + return pOperator; +} + +SOperatorInfo* createOffsetOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream) { + SOffsetOperatorInfo* pInfo = calloc(1, sizeof(SOffsetOperatorInfo)); + + pInfo->offset = pRuntimeEnv->pQuery->limit.offset; + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "OffsetOperator"; + pOperator->operatorType = OP_Offset; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->exec = doOffset; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + return pOperator; +} + +SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "TimeIntervalAggOperator"; + pOperator->operatorType = OP_TimeWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo)); + + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + pInfo->prevTs = INT64_MIN; + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "SessionWindowAggOperator"; + pOperator->operatorType = OP_SessionWindow; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = doSessionWindowAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); + + pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "MultiTableTimeIntervalOperator"; + pOperator->operatorType = OP_MultiTableTimeInterval; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doSTableIntervalAgg; + pOperator->cleanup = destroyBasicOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createGroupbyOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); + pInfo->colIndex = -1; // group by column index + + pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "GroupbyAggOperator"; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Groupby; + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->exec = hashGroupbyAggregate; + pOperator->cleanup = destroyGroupbyOperatorInfo; + + return pOperator; +} + +SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, + int32_t numOfOutput) { + SFillOperatorInfo* pInfo = calloc(1, sizeof(SFillOperatorInfo)); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + + { + SQuery* pQuery = pRuntimeEnv->pQuery; + SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQuery->fillVal); + STimeWindow w = TSWINDOW_INITIALIZER; + + TSKEY sk = MIN(pQuery->window.skey, pQuery->window.ekey); + TSKEY ek = MAX(pQuery->window.skey, pQuery->window.ekey); + getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w); + + pInfo->pFillInfo = taosCreateFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, + pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision, + pQuery->fillType, pColInfo, pRuntimeEnv->qinfo); } + + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + + pOperator->name = "FillOperator"; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->operatorType = OP_Fill; + + pOperator->upstream = upstream; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->info = pInfo; + pOperator->pRuntimeEnv = pRuntimeEnv; + + pOperator->exec = doFill; + pOperator->cleanup = destroySFillOperatorInfo; + + return pOperator; } -static void tableQueryImpl(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +static SSDataBlock* doTagScan(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } - if (queryHasRemainResForTableQuery(pRuntimeEnv)) { - if (pQuery->fillType != TSDB_FILL_NONE) { - /* - * There are remain results that are not returned due to result interpolation - * So, we do keep in this procedure instead of launching retrieve procedure for next results. - */ - int32_t numOfFilled = 0; - pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); + SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - if (pQuery->rec.rows > 0) { - limitResults(pRuntimeEnv); - } + int32_t maxNumOfTables = (int32_t)pRuntimeEnv->resultInfo.capacity; - qDebug("QInfo:%p current:%" PRId64 " returned, total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); - } else { - pQuery->rec.rows = 0; - assert(pRuntimeEnv->windowResInfo.size > 0); + STagScanInfo *pInfo = pOperator->info; + SSDataBlock *pRes = pInfo->pRes; - if (pQInfo->groupIndex < pRuntimeEnv->windowResInfo.size) { - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - } + int32_t count = 0; + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + + int32_t functionId = pOperator->pExpr[0].base.functionId; + if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id + SQuery* pQuery = pRuntimeEnv->pQuery; + assert(pQuery->numOfOutput == 1); + + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + int32_t rsize = pExprInfo->bytes; + + count = 0; + + int16_t bytes = pExprInfo->bytes; + int16_t type = pExprInfo->type; - if (pQuery->rec.rows > 0) { - qDebug("QInfo:%p %" PRId64 " rows returned from group results, total:%" PRId64 "", pQInfo, pQuery->rec.rows, - pQuery->rec.total); + for(int32_t i = 0; i < pQuery->numOfTags; ++i) { + if (pQuery->tagColList[i].colId == pExprInfo->base.colInfo.colId) { + bytes = pQuery->tagColList[i].bytes; + type = pQuery->tagColList[i].type; + break; } + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + + while(pInfo->currentIndex < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->currentIndex++; + STableQueryInfo *item = taosArrayGetP(pa, i); + + char *output = pColInfo->pData + count * rsize; + varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - // there are not data remains - if (pQuery->rec.rows <= 0 || pRuntimeEnv->windowResInfo.size <= pQInfo->groupIndex) { - qDebug("QInfo:%p query over, %" PRId64 " rows are returned", pQInfo, pQuery->rec.total); + output = varDataVal(output); + STableId* id = TSDB_TABLEID(item->pTable); + + *(int16_t *)output = 0; + output += sizeof(int16_t); + + *(int64_t *)output = id->uid; // memory align problem, todo serialize + output += sizeof(id->uid); + + *(int32_t *)output = id->tid; + output += sizeof(id->tid); + + *(int32_t *)output = pQuery->vgId; + output += sizeof(pQuery->vgId); + + char* data = NULL; + if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); } + + doSetTagValueToResultBuf(output, data, type, bytes); + count += 1; } - return; - } + qDebug("QInfo:%"PRIu64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); + } else if (functionId == TSDB_FUNC_COUNT) {// handle the "count(tbname)" query + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + *(int64_t*)pColInfo->pData = pInfo->totalTables; + count = 1; - // number of points returned during this query - pQuery->rec.rows = 0; - int64_t st = taosGetTimestampUs(); + pOperator->status = OP_EXEC_DONE; + qDebug("QInfo:%"PRIu64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); + } else { // return only the tags|table name etc. + SExprInfo* pExprInfo = pOperator->pExpr; // todo use the column list instead of exprinfo - assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); - SArray* g = GET_TABLEGROUP(pQInfo, 0); + count = 0; + while(pInfo->currentIndex < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->currentIndex++; - STableQueryInfo* item = taosArrayGetP(g, 0); - pQuery->current = item; + STableQueryInfo* item = taosArrayGetP(pa, i); - // group by normal column, sliding window query, interval query are handled by interval query processor - if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyColumn) { // interval (down sampling operation) - tableIntervalProcess(pQInfo, item); - } else if (isFixedOutputQuery(pRuntimeEnv)) { - tableAggregationProcess(pQInfo, item); - } else { // diff/add/multiply/subtract/division - assert(pQuery->checkResultBuf == 1); - tableProjectionProcess(pQInfo, item); + char *data = NULL, *dst = NULL; + int16_t type = 0, bytes = 0; + for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { + continue; + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); + type = pExprInfo[j].type; + bytes = pExprInfo[j].bytes; + + if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); + } + + dst = pColInfo->pData + count * pExprInfo[j].bytes; + doSetTagValueToResultBuf(dst, data, type, bytes); + } + + count += 1; + } + + if (pInfo->currentIndex >= pInfo->totalTables) { + pOperator->status = OP_EXEC_DONE; + } + + qDebug("QInfo:%"PRIu64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); } - // record the total elapsed time - pRuntimeEnv->summary.elapsedTime += (taosGetTimestampUs() - st); - assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); + pRes->info.rows = count; + return (pRes->info.rows == 0)? NULL:pInfo->pRes; } -static void stableQueryImpl(SQInfo *pQInfo) { - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pRuntimeEnv->pQuery; - pQuery->rec.rows = 0; +SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { + STagScanInfo* pInfo = calloc(1, sizeof(STagScanInfo)); + pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - int64_t st = taosGetTimestampUs(); + size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + assert(numOfGroup == 0 || numOfGroup == 1); - if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyColumn))) { - multiTableQueryProcess(pQInfo); - } else { - assert((pQuery->checkResultBuf == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || - pRuntimeEnv->groupbyColumn); + pInfo->totalTables = pRuntimeEnv->tableqinfoGroupInfo.numOfTables; + pInfo->currentIndex = 0; - sequentialTableProcess(pQInfo); - } + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + pOperator->name = "SeqTableTagScan"; + pOperator->operatorType = OP_TagScan; + pOperator->blockingOptr = false; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->exec = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanup = destroyTagScanOperatorInfo; - // record the total elapsed time - pQInfo->runtimeEnv.summary.elapsedTime += (taosGetTimestampUs() - st); + return pOperator; } static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) { @@ -5866,7 +5386,10 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { if (pExprMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { return TSDB_TBNAME_COLUMN_INDEX; + } else if (pExprMsg->colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + return TSDB_BLOCK_DIST_COLUMN_INDEX; } + while(j < pQueryMsg->numOfTags) { if (pExprMsg->colInfo.colId == pTagCols[j].colId) { @@ -5902,6 +5425,17 @@ static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) { return false; } + if (pQueryMsg->sw.gap < 0 || pQueryMsg->sw.primaryColId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + qError("qmsg:%p illegal value of session window time %" PRId64, pQueryMsg, pQueryMsg->sw.gap); + return false; + } + + if (pQueryMsg->sw.gap > 0 && pQueryMsg->interval.interval > 0) { + qError("qmsg:%p illegal value of session window time %" PRId64" and interval value %"PRId64, pQueryMsg, + pQueryMsg->sw.gap, pQueryMsg->interval.interval); + return false; + } + if (pQueryMsg->numOfTables <= 0) { qError("qmsg:%p illegal value of numOfTables %d", pQueryMsg, pQueryMsg->numOfTables); return false; @@ -5933,7 +5467,8 @@ static bool validateQuerySourceCols(SQueryTableMsg *pQueryMsg, SSqlFuncMsg** pEx if ((pFuncMsg->functionId == TSDB_FUNC_TAGPRJ) || (pFuncMsg->functionId == TSDB_FUNC_TID_TAG && pFuncMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || - (pFuncMsg->functionId == TSDB_FUNC_COUNT && pFuncMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) { + (pFuncMsg->functionId == TSDB_FUNC_COUNT && pFuncMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) || + (pFuncMsg->functionId == TSDB_FUNC_BLKINFO)) { continue; } @@ -5977,8 +5512,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @param pExpr * @return */ -static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, SSqlFuncMsg ***pSecStageExpr, - char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols, char** sql) { +int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) { int32_t code = TSDB_CODE_SUCCESS; if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { @@ -6013,6 +5547,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); + pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); + pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); + pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); // query msg safety check if (!validateQueryMsg(pQueryMsg)) { @@ -6073,8 +5610,8 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } - *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); - if (*pExpr == NULL) { + param->pExprMsg = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (param->pExprMsg == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } @@ -6082,11 +5619,14 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { - (*pExpr)[i] = pExprMsg; + param->pExprMsg[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->colType = htons(pExprMsg->colType); + pExprMsg->colBytes = htons(pExprMsg->colBytes); + pExprMsg->functionId = htons(pExprMsg->functionId); pExprMsg->numOfParams = htons(pExprMsg->numOfParams); pExprMsg->resColId = htons(pExprMsg->resColId); @@ -6118,14 +5658,17 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pQueryMsg->secondStageOutput) { pExprMsg = (SSqlFuncMsg *)pMsg; - *pSecStageExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); + param->pSecExprMsg = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { - (*pSecStageExpr)[i] = pExprMsg; + param->pSecExprMsg[i] = pExprMsg; pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); - pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); + pExprMsg->colType = htons(pExprMsg->colType); + pExprMsg->colBytes = htons(pExprMsg->colBytes); + pExprMsg->functionId = htons(pExprMsg->functionId); pExprMsg->numOfParams = htons(pExprMsg->numOfParams); @@ -6155,27 +5698,27 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } } - pMsg = createTableIdList(pQueryMsg, pMsg, pTableIdList); + pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns - *groupbyCols = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); - if (*groupbyCols == NULL) { + param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); + if (param->pGroupColIndex == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { - (*groupbyCols)[i].colId = htons(*(int16_t *)pMsg); - pMsg += sizeof((*groupbyCols)[i].colId); + param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].colId); - (*groupbyCols)[i].colIndex = htons(*(int16_t *)pMsg); - pMsg += sizeof((*groupbyCols)[i].colIndex); + param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].colIndex); - (*groupbyCols)[i].flag = htons(*(int16_t *)pMsg); - pMsg += sizeof((*groupbyCols)[i].flag); + param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); + pMsg += sizeof(param->pGroupColIndex[i].flag); - memcpy((*groupbyCols)[i].name, pMsg, tListLen(groupbyCols[i]->name)); - pMsg += tListLen((*groupbyCols)[i].name); + memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); + pMsg += tListLen(param->pGroupColIndex[i].name); } pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); @@ -6195,8 +5738,8 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } if (pQueryMsg->numOfTags > 0) { - (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); - if (*tagCols == NULL) { + param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (param->pTagColumnInfo == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } @@ -6209,38 +5752,53 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pTagCol->type = htons(pTagCol->type); pTagCol->numOfFilters = 0; - (*tagCols)[i] = *pTagCol; + param->pTagColumnInfo[i] = *pTagCol; pMsg += sizeof(SColumnInfo); } } // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { - *tagCond = calloc(1, pQueryMsg->tagCondLen); - - if (*tagCond == NULL) { + param->tagCond = calloc(1, pQueryMsg->tagCondLen); + if (param->tagCond == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; - } - memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); + + memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); pMsg += pQueryMsg->tagCondLen; } + if (pQueryMsg->prevResultLen > 0) { + param->prevResult = calloc(1, pQueryMsg->prevResultLen); + if (param->prevResult == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); + pMsg += pQueryMsg->prevResultLen; + } + if (pQueryMsg->tbnameCondLen > 0) { - *tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); - if (*tbnameCond == NULL) { + param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); + if (param->tbnameCond == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _cleanup; } - strncpy(*tbnameCond, pMsg, pQueryMsg->tbnameCondLen); + strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); pMsg += pQueryMsg->tbnameCondLen; } - *sql = strndup(pMsg, pQueryMsg->sqlstrLen); + //skip ts buf + if ((pQueryMsg->tsOffset + pQueryMsg->tsLen) > 0) { + pMsg = (char *)pQueryMsg + pQueryMsg->tsOffset + pQueryMsg->tsLen; + } + + param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); - if (!validateQuerySourceCols(pQueryMsg, *pExpr, *tagCols)) { + if (!validateQuerySourceCols(pQueryMsg, param->pExprMsg, param->pTagColumnInfo)) { code = TSDB_CODE_QRY_INVALID_MSG; goto _cleanup; } @@ -6251,19 +5809,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); - qDebug("qmsg:%p, sql:%s", pQueryMsg, *sql); + qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); return TSDB_CODE_SUCCESS; _cleanup: - tfree(*pExpr); - taosArrayDestroy(*pTableIdList); - *pTableIdList = NULL; - tfree(*tbnameCond); - tfree(*groupbyCols); - tfree(*tagCols); - tfree(*tagCond); - tfree(*sql); - + freeParam(param); return code; } @@ -6288,13 +5838,34 @@ static int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, SQueryTable return TSDB_CODE_SUCCESS; } -static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, SSqlFuncMsg **pExprMsg, - SColumnInfo* pTagCols) { - *pExprInfo = NULL; - int32_t code = TSDB_CODE_SUCCESS; +static int32_t updateOutputBufForTopBotQuery(SQueryTableMsg* pQueryMsg, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { + for (int32_t i = 0; i < numOfOutput; ++i) { + int16_t functId = pExprs[i].base.functionId; - SExprInfo *pExprs = (SExprInfo *)calloc(pQueryMsg->numOfOutput, sizeof(SExprInfo)); - if (pExprs == NULL) { + if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { + int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); + if (j < 0 || j >= pQueryMsg->numOfCols) { + return TSDB_CODE_QRY_INVALID_MSG; + } else { + SColumnInfo* pCol = &pQueryMsg->colList[j]; + int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, superTable); + assert(ret == TSDB_CODE_SUCCESS); + } + } + } + + return TSDB_CODE_SUCCESS; +} + +// TODO tag length should be passed from client +int32_t createQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlFuncMsg** pExprMsg, SColumnInfo* pTagCols) { + *pExprInfo = NULL; + int32_t code = TSDB_CODE_SUCCESS; + + SExprInfo *pExprs = (SExprInfo *)calloc(pQueryMsg->numOfOutput, sizeof(SExprInfo)); + if (pExprs == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -6318,9 +5889,13 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num } type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypeDesc[type].nSize; + bytes = tDataTypes[type].bytes; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column - SSchema s = tGetTableNameColumnSchema(); + SSchema* s = tGetTbnameColumnSchema(); + type = s->type; + bytes = s->bytes; + } else if (pExprs[i].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + SSchema s = tGetBlockDistColumnSchema(); type = s.type; bytes = s.bytes; } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX) { @@ -6336,7 +5911,7 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); if (TSDB_COL_IS_TAG(pExprs[i].base.colInfo.flag)) { - if (j < TSDB_TBNAME_COLUMN_INDEX || j >= pQueryMsg->numOfTags) { + if (j < TSDB_BLOCK_DIST_COLUMN_INDEX || j >= pQueryMsg->numOfTags) { return TSDB_CODE_QRY_INVALID_MSG; } } else { @@ -6350,14 +5925,20 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num type = pCol->type; bytes = pCol->bytes; } else { - SSchema s = tGetTableNameColumnSchema(); + SSchema* s = tGetTbnameColumnSchema(); - type = s.type; - bytes = s.bytes; + type = s->type; + bytes = s->bytes; } } int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64; + if (pExprs[i].base.functionId != TSDB_FUNC_ARITHM && + (type != pExprs[i].base.colType || bytes != pExprs[i].base.colBytes)) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; + } + if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { tfree(pExprs); @@ -6371,30 +5952,66 @@ static int32_t createQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t num assert(isValidDataType(pExprs[i].type)); } - // TODO refactor + // the tag length is affected by other tag columns, so this should be update. + updateOutputBufForTopBotQuery(pQueryMsg, pTagCols, pExprs, numOfOutput, tagLen, isSuperTable); + + *pExprInfo = pExprs; + return TSDB_CODE_SUCCESS; +} + +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, + SSqlFuncMsg **pExprMsg, SExprInfo *prevExpr) { + *pExprInfo = NULL; + int32_t code = TSDB_CODE_SUCCESS; + + SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); + if (pExprs == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); + for (int32_t i = 0; i < numOfOutput; ++i) { pExprs[i].base = *pExprMsg[i]; - int16_t functId = pExprs[i].base.functionId; + pExprs[i].bytes = 0; - if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { - int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - if (j < 0 || j >= pQueryMsg->numOfCols) { - assert(0); - } else { - SColumnInfo *pCol = &pQueryMsg->colList[j]; - int32_t ret = - getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, - &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); - assert(ret == TSDB_CODE_SUCCESS); + int16_t type = 0; + int16_t bytes = 0; + + // parse the arithmetic expression + if (pExprs[i].base.functionId == TSDB_FUNC_ARITHM) { + code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg); + + if (code != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return code; } + + type = TSDB_DATA_TYPE_DOUBLE; + bytes = tDataTypes[type].bytes; + } else { + int32_t index = pExprs[i].base.colInfo.colIndex; + assert(prevExpr[index].base.resColId == pExprs[i].base.colInfo.colId); + + type = prevExpr[index].type; + bytes = prevExpr[index].bytes; + } + + int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64; + if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, + &pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { + tfree(pExprs); + return TSDB_CODE_QRY_INVALID_MSG; } + + assert(isValidDataType(pExprs[i].type)); } *pExprInfo = pExprs; return TSDB_CODE_SUCCESS; } -static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { +SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; } @@ -6418,7 +6035,7 @@ static SSqlGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SCol return pGroupbyExpr; } -static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { +static int32_t createFilterInfo(SQuery *pQuery, uint64_t qId) { for (int32_t i = 0; i < pQuery->numOfCols; ++i) { if (pQuery->colList[i].numOfFilters > 0) { pQuery->numOfFilterCols++; @@ -6454,13 +6071,13 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { int32_t lower = pSingleColFilter->filterInfo.lowerRelOptr; int32_t upper = pSingleColFilter->filterInfo.upperRelOptr; if (lower == TSDB_RELATION_INVALID && upper == TSDB_RELATION_INVALID) { - qError("QInfo:%p invalid filter info", pQInfo); + qError("QInfo:%"PRIu64" invalid filter info", qId); return TSDB_CODE_QRY_INVALID_MSG; } pSingleColFilter->fp = getFilterOperator(lower, upper); if (pSingleColFilter->fp == NULL) { - qError("QInfo:%p invalid filter info", pQInfo); + qError("QInfo:%"PRIu64" invalid filter info", qId); return TSDB_CODE_QRY_INVALID_MSG; } @@ -6497,6 +6114,9 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { assert(f < pQuery->numOfCols); } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { // do nothing for user-defined constant value result columns + } else if (pColIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX) { + pColIndex->colIndex = 0;// only one source column, so it must be 0; + assert(pQuery->numOfOutput == 1); } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { @@ -6506,34 +6126,42 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } - assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); + assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX || pColIndex->colId == TSDB_BLOCK_DIST_COLUMN_INDEX); } } } -static void freeQInfo(SQInfo *pQInfo); +void setResultBufSize(SQuery* pQuery, SRspResultInfo* pResultInfo) { + const int32_t DEFAULT_RESULT_MSG_SIZE = 1024 * (1024 + 512); -static void calResultBufSize(SQuery* pQuery) { - const int32_t RESULT_MSG_MIN_SIZE = 1024 * (1024 + 512); // bytes - const int32_t RESULT_MSG_MIN_ROWS = 8192; - const float RESULT_THRESHOLD_RATIO = 0.85f; + // the minimum number of rows for projection query + const int32_t MIN_ROWS_FOR_PRJ_QUERY = 8192; + const int32_t DEFAULT_MIN_ROWS = 4096; + + const float THRESHOLD_RATIO = 0.85f; if (isProjQuery(pQuery)) { - int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize; - if (numOfRes < RESULT_MSG_MIN_ROWS) { - numOfRes = RESULT_MSG_MIN_ROWS; + int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQuery->resultRowSize; + if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { + numOfRes = MIN_ROWS_FOR_PRJ_QUERY; } - pQuery->rec.capacity = numOfRes; - pQuery->rec.threshold = (int32_t)(numOfRes * RESULT_THRESHOLD_RATIO); + pResultInfo->capacity = numOfRes; } else { // in case of non-prj query, a smaller output buffer will be used. - pQuery->rec.capacity = 4096; - pQuery->rec.threshold = (int32_t)(pQuery->rec.capacity * RESULT_THRESHOLD_RATIO); + pResultInfo->capacity = DEFAULT_MIN_ROWS; } + + pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); + pResultInfo->total = 0; +} + +FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { + return ((SQInfo *)qHandle)->qId == qId; } -static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, char* sql) { +SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SSqlGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, + SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, bool stableQuery, + char* sql, uint64_t *qId) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -6544,15 +6172,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; - pQInfo->tableGroupInfo = *pTableGroupInfo; - - SQuery *pQuery = calloc(1, sizeof(SQuery)); - if (pQuery == NULL) { - goto _cleanup_query; - } - + SQuery* pQuery = &pQInfo->query; pQInfo->runtimeEnv.pQuery = pQuery; + pQuery->tableGroupInfo = *pTableGroupInfo; pQuery->numOfCols = numOfCols; pQuery->numOfOutput = numOfOutput; pQuery->limit.limit = pQueryMsg->limit; @@ -6569,57 +6192,41 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->tagColList = pTagCols; pQuery->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; pQuery->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; - + pQuery->sw = pQueryMsg->sw; pQuery->colList = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); if (pQuery->colList == NULL) { goto _cleanup; } - int32_t srcSize = 0; + pQuery->srcRowSize = 0; + pQuery->maxSrcColumnSize = 0; for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i] = pQueryMsg->colList[i]; - pQuery->colList[i].filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pQuery->colList[i].numOfFilters); - srcSize += pQuery->colList[i].bytes; + pQuery->colList[i].filters = tFilterInfoDup(pQueryMsg->colList[i].filters, pQuery->colList[i].numOfFilters); + + pQuery->srcRowSize += pQuery->colList[i].bytes; + if (pQuery->maxSrcColumnSize < pQuery->colList[i].bytes) { + pQuery->maxSrcColumnSize = pQuery->colList[i].bytes; + } } // calculate the result row size for (int16_t col = 0; col < numOfOutput; ++col) { assert(pExprs[col].bytes > 0); - pQuery->rowSize += pExprs[col].bytes; + pQuery->resultRowSize += pExprs[col].bytes; + + // keep the tag length + if (TSDB_COL_IS_TAG(pExprs[col].base.colInfo.flag)) { + pQuery->tagLen += pExprs[col].bytes; + } } doUpdateExprColumnIndex(pQuery); - - int32_t ret = createFilterInfo(pQInfo, pQuery); + int32_t ret = createFilterInfo(pQuery, pQInfo->qId); if (ret != TSDB_CODE_SUCCESS) { goto _cleanup; } - // prepare the result buffer - pQuery->sdata = (tFilePage **)calloc(pQuery->numOfOutput, POINTER_BYTES); - if (pQuery->sdata == NULL) { - goto _cleanup; - } - - calResultBufSize(pQuery); - - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - // allocate additional memory for interResults that are usually larger then final results - // TODO refactor - int16_t bytes = 0; - if (pQuery->pExpr2 == NULL || col > pQuery->numOfExpr2) { - bytes = pExprs[col].bytes; - } else { - bytes = MAX(pQuery->pExpr2[col].bytes, pExprs[col].bytes); - } - - size_t size = (size_t)((pQuery->rec.capacity + 1) * bytes + pExprs[col].interBytes + sizeof(tFilePage)); - pQuery->sdata[col] = (tFilePage *)calloc(1, size); - if (pQuery->sdata[col] == NULL) { - goto _cleanup; - } - } - if (pQuery->fillType != TSDB_FILL_NONE) { pQuery->fillVal = malloc(sizeof(int64_t) * pQuery->numOfOutput); if (pQuery->fillVal == NULL) { @@ -6633,26 +6240,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou size_t numOfGroups = 0; if (pTableGroupInfo->pGroupList != NULL) { numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); + STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; - pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); - pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables; - pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - - pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); - pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); - - pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); - pQInfo->runtimeEnv.pool = initResultRowPool(getResultRowSize(&pQInfo->runtimeEnv)); - pQInfo->runtimeEnv.prevRow = malloc(POINTER_BYTES * pQuery->numOfCols + srcSize); - - char* start = POINTER_BYTES * pQuery->numOfCols + (char*) pQInfo->runtimeEnv.prevRow; - pQInfo->runtimeEnv.prevRow[0] = start; - - for(int32_t i = 1; i < pQuery->numOfCols; ++i) { - pQInfo->runtimeEnv.prevRow[i] = pQInfo->runtimeEnv.prevRow[i - 1] + pQuery->colList[i-1].bytes; + pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); + pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; + pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); @@ -6660,24 +6252,23 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou goto _cleanup; } - // NOTE: pTableCheckInfo need to update the query time range and the lastKey info - pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; pQInfo->sql = sql; pthread_mutex_init(&pQInfo->lock, NULL); tsem_init(&pQInfo->ready, 0, 0); - pQuery->pos = -1; pQuery->window = pQueryMsg->window; changeExecuteScanOrder(pQInfo, pQueryMsg, stableQuery); - pQInfo->runtimeEnv.queryWindowIdentical = true; + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + bool groupByCol = isGroupbyColumn(pQuery->pGroupbyExpr); + STimeWindow window = pQuery->window; int32_t index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* pa = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + SArray* pa = taosArrayGetP(pQuery->tableGroupInfo.pGroupList, i); size_t s = taosArrayGetSize(pa); SArray* p1 = taosArrayInit(s, POINTER_BYTES); @@ -6685,18 +6276,14 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou goto _cleanup; } - taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); + taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - window.skey = info->lastKey; - if (info->lastKey != pQuery->window.skey) { - pQInfo->runtimeEnv.queryWindowIdentical = false; - } void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); - STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); + STableQueryInfo* item = createTableQueryInfo(pQuery, info->pTable, groupByCol, window, buf); if (item == NULL) { goto _cleanup; } @@ -6705,20 +6292,24 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou taosArrayPush(p1, &item); STableId* id = TSDB_TABLEID(info->pTable); - taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); + taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); index += 1; } } - colIdCheck(pQuery); + colIdCheck(pQuery, pQInfo->qId); - qDebug("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo); + // todo refactor + pQInfo->query.queryBlockDist = (numOfOutput == 1 && pExprs[0].base.colInfo.colId == TSDB_BLOCK_DIST_COLUMN_INDEX); + + pQInfo->qId = atomic_add_fetch_64(&queryHandleId, 1); + *qId = pQInfo->qId; + qDebug("qmsg:%p QInfo:%" PRIu64 "-%p created", pQueryMsg, pQInfo->qId, pQInfo); return pQInfo; _cleanup_qinfo: tsdbDestroyTableGroup(pTableGroupInfo); -_cleanup_query: if (pGroupbyExpr != NULL) { taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); @@ -6728,7 +6319,8 @@ _cleanup_query: for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExprs[i]; if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(&pExprInfo->pExpr, NULL); + tExprTreeDestroy(pExprInfo->pExpr, NULL); + pExprInfo->pExpr = NULL; } } @@ -6739,7 +6331,7 @@ _cleanup: return NULL; } -static bool isValidQInfo(void *param) { +bool isValidQInfo(void *param) { SQInfo *pQInfo = (SQInfo *)param; if (pQInfo == NULL) { return false; @@ -6753,40 +6345,50 @@ static bool isValidQInfo(void *param) { return (sig == (uint64_t)pQInfo); } -static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) { +int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, SQueryParam* param, bool isSTable) { int32_t code = TSDB_CODE_SUCCESS; - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - STSBuf *pTSBuf = NULL; + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + pRuntimeEnv->qinfo = pQInfo; + + SQuery *pQuery = pRuntimeEnv->pQuery; + + STSBuf *pTsBuf = NULL; if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset; - pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); + pTsBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); - tsBufResetPos(pTSBuf); - bool ret = tsBufNextPos(pTSBuf); + tsBufResetPos(pTsBuf); + bool ret = tsBufNextPos(pTsBuf); UNUSED(ret); } + SArray* prevResult = NULL; + if (pQueryMsg->prevResultLen > 0) { + prevResult = interResFromBinary(param->prevResult, pQueryMsg->prevResultLen); + } + pQuery->precision = tsdbGetCfg(tsdb)->precision; if ((QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.skey > pQuery->window.ekey)) || (!QUERY_IS_ASC_QUERY(pQuery) && (pQuery->window.ekey > pQuery->window.skey))) { - qDebug("QInfo:%p no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo, pQuery->window.skey, + qDebug("QInfo:%"PRIu64" no result in time range %" PRId64 "-%" PRId64 ", order %d", pQInfo->qId, pQuery->window.skey, pQuery->window.ekey, pQuery->order.order); - setQueryStatus(pQuery, QUERY_COMPLETED); - pQInfo->tableqinfoGroupInfo.numOfTables = 0; + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + pRuntimeEnv->tableqinfoGroupInfo.numOfTables = 0; + // todo free memory return TSDB_CODE_SUCCESS; } - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); - setQueryStatus(pQuery, QUERY_COMPLETED); + if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { + qDebug("QInfo:%"PRIu64" no table qualified for tag filter, abort query", pQInfo->qId); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); return TSDB_CODE_SUCCESS; } // filter the qualified - if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { + if ((code = doInitQInfo(pQInfo, pTsBuf, prevResult, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } @@ -6798,7 +6400,7 @@ _error: return code; } -static void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters) { if (pFilter == NULL || numOfFilters == 0) { return; } @@ -6844,7 +6446,7 @@ static void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { for (int32_t i = 0; i < numOfExpr; ++i) { if (pExprInfo[i].pExpr != NULL) { - tExprNodeDestroy(pExprInfo[i].pExpr, NULL); + tExprTreeDestroy(pExprInfo[i].pExpr, NULL); } } @@ -6852,26 +6454,19 @@ static void* destroyQueryFuncExpr(SExprInfo* pExprInfo, int32_t numOfExpr) { return NULL; } -static void freeQInfo(SQInfo *pQInfo) { +void freeQInfo(SQInfo *pQInfo) { if (!isValidQInfo(pQInfo)) { return; } - qDebug("QInfo:%p start to free QInfo", pQInfo); - - releaseQueryBuf(pQInfo->tableqinfoGroupInfo.numOfTables); + qDebug("QInfo:%"PRIu64" start to free QInfo", pQInfo->qId); + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (pQuery != NULL) { - if (pQuery->sdata != NULL) { - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - tfree(pQuery->sdata[col]); - } - tfree(pQuery->sdata); - } - if (pQuery->fillVal != NULL) { tfree(pQuery->fillVal); } @@ -6901,266 +6496,81 @@ static void freeQInfo(SQInfo *pQInfo) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); tfree(pQuery->pGroupbyExpr); } - - tfree(pQuery); } - doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo); + doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); tfree(pQInfo->pBuf); tfree(pQInfo->sql); - tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); - taosHashCleanup(pQInfo->arrTableIdInfo); - - taosArrayDestroy(pQInfo->groupResInfo.pRows); + tsdbDestroyTableGroup(&pQuery->tableGroupInfo); + taosArrayDestroy(pRuntimeEnv->groupResInfo.pRows); pQInfo->signature = 0; - qDebug("QInfo:%p QInfo is freed", pQInfo); + qDebug("QInfo:%"PRIu64" QInfo is freed", pQInfo->qId); tfree(pQInfo); } -static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - /* - * get the file size and set the numOfRows to be the file size, since for tsComp query, - * the returned row size is equalled to 1 - * TODO handle the case that the file is too large to send back one time - */ - if (isTSCompQuery(pQuery) && (*numOfRows) > 0) { - struct stat fStat; - FILE *f = *(FILE **)pQuery->sdata[0]->data; - if ((f != NULL) && (fstat(fileno(f), &fStat) == 0)) { - *numOfRows = fStat.st_size; - return fStat.st_size; - } else { - qError("QInfo:%p failed to get file info, file:%p, reason:%s", pQInfo, f, strerror(errno)); - return 0; - } - } else { - return (size_t)(pQuery->rowSize * (*numOfRows)); - } -} - -static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { +int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // the remained number of retrieved rows, not the interpolated result + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; // load data from file to msg buffer - if (isTSCompQuery(pQuery)) { - - FILE *f = *(FILE **)pQuery->sdata[0]->data; // TODO refactor + if (isTsCompQuery(pQuery)) { + SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); + FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor // make sure file exist if (f) { off_t s = lseek(fileno(f), 0, SEEK_END); + assert(s == pRuntimeEnv->outputBuf->info.rows); - qDebug("QInfo:%p ts comp data return, file:%p, size:%"PRId64, pQInfo, f, s); + qDebug("QInfo:%"PRIu64" ts comp data return, file:%p, size:%"PRId64, pQInfo->qId, f, (uint64_t)s); if (fseek(f, 0, SEEK_SET) >= 0) { size_t sz = fread(data, 1, s, f); if(sz < s) { // todo handle error + qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s); assert(0); } } else { UNUSED(s); + qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno)); + assert(0); + } + + // dump error info + if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) { + qDump(data, s); + assert(0); } fclose(f); - *(FILE **)pQuery->sdata[0]->data = NULL; } // all data returned, set query over - if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - setQueryStatus(pQuery, QUERY_OVER); + if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { + setQueryStatus(pRuntimeEnv, QUERY_OVER); } } else { - doCopyQueryResultToMsg(pQInfo, (int32_t)pQuery->rec.rows, data); + doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data); } - pQuery->rec.total += pQuery->rec.rows; - qDebug("QInfo:%p current numOfRes rows:%" PRId64 ", total:%" PRId64, pQInfo, pQuery->rec.rows, pQuery->rec.total); + pRuntimeEnv->resultInfo.total += pRuntimeEnv->outputBuf->info.rows; + qDebug("QInfo:%"PRIu64" current numOfRes rows:%d, total:%" PRId64, pQInfo->qId, + pRuntimeEnv->outputBuf->info.rows, pRuntimeEnv->resultInfo.total); - if (pQuery->limit.limit > 0 && pQuery->limit.limit == pQuery->rec.total) { - qDebug("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit); - setQueryStatus(pQuery, QUERY_OVER); + if (pQuery->limit.limit > 0 && pQuery->limit.limit == pRuntimeEnv->resultInfo.total) { + qDebug("QInfo:%"PRIu64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQuery->limit.limit); + setQueryStatus(pRuntimeEnv, QUERY_OVER); } return TSDB_CODE_SUCCESS; } -typedef struct SQueryMgmt { - SCacheObj *qinfoPool; // query handle pool - int32_t vgId; - bool closed; - pthread_mutex_t lock; -} SQueryMgmt; - -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo) { - assert(pQueryMsg != NULL && tsdb != NULL); - - int32_t code = TSDB_CODE_SUCCESS; - - char *sql = NULL; - char *tagCond = NULL; - char *tbnameCond = NULL; - SArray *pTableIdList = NULL; - SSqlFuncMsg **pExprMsg = NULL; - SSqlFuncMsg **pSecExprMsg = NULL; - SExprInfo *pExprs = NULL; - SExprInfo *pSecExprs = NULL; - - SColIndex *pGroupColIndex = NULL; - SColumnInfo *pTagColumnInfo = NULL; - SSqlGroupbyExpr *pGroupbyExpr = NULL; - - code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &pSecExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo, &sql); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (pQueryMsg->numOfTables <= 0) { - qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if (pTableIdList == NULL || taosArrayGetSize(pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->numOfOutput, &pExprs, pExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (pSecExprMsg != NULL) { - if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &pSecExprs, pSecExprMsg, pTagColumnInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, pGroupColIndex, &code); - if ((pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _over; - } - - bool isSTableQuery = false; - STableGroupInfo tableGroupInfo = {0}; - int64_t st = taosGetTimestampUs(); - - if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(pTableIdList, 0); - - qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { - isSTableQuery = true; - - // also note there's possibility that only one table in the super table - if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, tagCond, pQueryMsg->tagCondLen, - pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, numOfGroupByCols); - - if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); - goto _over; - } - } else { - code = tsdbGetTableGroupFromIdList(tsdb, pTableIdList, &tableGroupInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - qDebug("qmsg:%p query on %" PRIzu " tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); - } - - int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%" PRIzu ", elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); - } else { - assert(0); - } - - code = checkForQueryBuf(tableGroupInfo.numOfTables); - if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort - goto _over; - } - - (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, pSecExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery, sql); - - sql = NULL; - pExprs = NULL; - pSecExprs = NULL; - pGroupbyExpr = NULL; - pTagColumnInfo = NULL; - - if ((*pQInfo) == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _over; - } - - code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery); - -_over: - free(tagCond); - free(tbnameCond); - free(pGroupColIndex); - - if (pGroupbyExpr != NULL) { - taosArrayDestroy(pGroupbyExpr->columnInfo); - free(pGroupbyExpr); - } - - free(pTagColumnInfo); - free(sql); - free(pExprs); - free(pSecExprs); - - free(pExprMsg); - free(pSecExprMsg); - - taosArrayDestroy(pTableIdList); - - for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { - SColumnInfo* column = pQueryMsg->colList + i; - freeColumnFilterInfo(column->filters, column->numOfFilters); - } - - //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; - if (code != TSDB_CODE_SUCCESS) { - *pQInfo = NULL; - } - - // if failed to add ref for all tables in this query, abort current query - return code; -} - -void qDestroyQueryInfo(qinfo_t qHandle) { - SQInfo* pQInfo = (SQInfo*) qHandle; - if (!isValidQInfo(pQInfo)) { - return; - } - - qDebug("QInfo:%p query completed", pQInfo); - queryCostStatis(pQInfo); // print the query cost summary - freeQInfo(pQInfo); -} - -static bool doBuildResCheck(SQInfo* pQInfo) { +bool doBuildResCheck(SQInfo* pQInfo) { bool buildRes = false; pthread_mutex_lock(&pQInfo->lock); @@ -7180,340 +6590,24 @@ static bool doBuildResCheck(SQInfo* pQInfo) { return buildRes; } -bool qTableQuery(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - assert(pQInfo && pQInfo->signature == pQInfo); - int64_t threadId = taosGetSelfPthreadId(); - - int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { - qError("QInfo:%p qhandle is now executed by thread:%p", pQInfo, (void*) curOwner); - pQInfo->code = TSDB_CODE_QRY_IN_EXEC; - return false; - } - - pQInfo->startExecTs = taosGetTimestampSec(); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:%p it is already killed, abort", pQInfo); - return doBuildResCheck(pQInfo); - } - - if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:%p no table exists for query, abort", pQInfo); - setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); - return doBuildResCheck(pQInfo); - } - - // error occurs, record the error code and return to client - int32_t ret = setjmp(pQInfo->runtimeEnv.env); - if (ret != TSDB_CODE_SUCCESS) { - pQInfo->code = ret; - qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code)); - return doBuildResCheck(pQInfo); - } - - qDebug("QInfo:%p query task is launched", pQInfo); - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) { - assert(pQInfo->runtimeEnv.pQueryHandle == NULL); - buildTagQueryResult(pQInfo); - } else if (pQInfo->runtimeEnv.stableQuery) { - stableQueryImpl(pQInfo); - } else { - tableQueryImpl(pQInfo); - } - - SQuery* pQuery = pRuntimeEnv->pQuery; - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:%p query is killed", pQInfo); - } else if (pQuery->rec.rows == 0) { - qDebug("QInfo:%p over, %" PRIzu " tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total); - } else { - qDebug("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", - pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); - } - - return doBuildResCheck(pQInfo); -} - -int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - qError("QInfo:%p invalid qhandle", pQInfo); - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - *buildRes = false; - if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:%p query is killed, code:0x%08x", pQInfo, pQInfo->code); - return pQInfo->code; - } - - int32_t code = TSDB_CODE_SUCCESS; - - if (tsRetrieveBlockingModel) { - pQInfo->rspContext = pRspContext; - tsem_wait(&pQInfo->ready); - *buildRes = true; - code = pQInfo->code; - } else { - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - pthread_mutex_lock(&pQInfo->lock); - - assert(pQInfo->rspContext == NULL); - if (pQInfo->dataReady == QUERY_RESULT_READY) { - *buildRes = true; - qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%" PRId64 ", code:%s", pQInfo, pQuery->rowSize, - pQuery->rec.rows, tstrerror(pQInfo->code)); - } else { - *buildRes = false; - qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo); - pQInfo->rspContext = pRspContext; - assert(pQInfo->rspContext != NULL); - } - - code = pQInfo->code; - pthread_mutex_unlock(&pQInfo->lock); - } - - return code; -} - -int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - size_t size = getResultSize(pQInfo, &pQuery->rec.rows); - - size += sizeof(int32_t); - size += sizeof(STableIdInfo) * taosHashGetSize(pQInfo->arrTableIdInfo); - - *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); - - // todo proper handle failed to allocate memory, - // current solution only avoid crash, but cannot return error code to client - *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); - if (*pRsp == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - (*pRsp)->numOfRows = htonl((int32_t)pQuery->rec.rows); - - if (pQInfo->code == TSDB_CODE_SUCCESS) { - (*pRsp)->offset = htobe64(pQuery->limit.offset); - (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); - } else { - (*pRsp)->offset = 0; - (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); - } - - (*pRsp)->precision = htons(pQuery->precision); - if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { - doDumpQueryResult(pQInfo, (*pRsp)->data); - } else { - setQueryStatus(pQuery, QUERY_OVER); - } - - pQInfo->rspContext = NULL; - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - - if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - // here current thread hold the refcount, so it is safe to free tsdbQueryHandle. - *continueExec = false; - (*pRsp)->completed = 1; // notify no more result to client - } else { - *continueExec = true; - qDebug("QInfo:%p has more results to retrieve", pQInfo); - } - - return pQInfo->code; -} - -int32_t qQueryCompleted(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER); -} - -int32_t qKillQuery(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - setQueryKilled(pQInfo); - - // Wait for the query executing thread being stopped/ - // Once the query is stopped, the owner of qHandle will be cleared immediately. - while (pQInfo->owner != 0) { - taosMsleep(100); - } - - return TSDB_CODE_SUCCESS; -} - static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - if (val == NULL) { - setVardataNull(output, type); - } else { - memcpy(output, val, varDataTLen(val)); - } - } else { - if (val == NULL) { - setNull(output, type, bytes); - } else { // todo here stop will cause client crash - memcpy(output, val, bytes); - } - } -} - -static void buildTagQueryResult(SQInfo* pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; - - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); - assert(numOfGroup == 0 || numOfGroup == 1); - - if (numOfGroup == 0) { + if (val == NULL) { + setNull(output, type, bytes); return; } - SArray* pa = GET_TABLEGROUP(pQInfo, 0); - - size_t num = taosArrayGetSize(pa); - assert(num == pQInfo->tableqinfoGroupInfo.numOfTables); - - int32_t count = 0; - int32_t functionId = pQuery->pExpr1[0].base.functionId; - if (functionId == TSDB_FUNC_TID_TAG) { // return the tags & table Id - assert(pQuery->numOfOutput == 1); - - SExprInfo* pExprInfo = &pQuery->pExpr1[0]; - int32_t rsize = pExprInfo->bytes; - count = 0; - - int16_t bytes = pExprInfo->bytes; - int16_t type = pExprInfo->type; - - for(int32_t i = 0; i < pQuery->numOfTags; ++i) { - if (pQuery->tagColList[i].colId == pExprInfo->base.colInfo.colId) { - bytes = pQuery->tagColList[i].bytes; - type = pQuery->tagColList[i].type; - break; - } - } - - while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) { - int32_t i = pQInfo->tableIndex++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pQuery->sdata[0]->data + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQInfo->vgId; - output += sizeof(pQInfo->vgId); - - if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - char* data = tsdbGetTableName(item->pTable); - memcpy(output, data, varDataTLen(data)); - } else { - char* data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); - doSetTagValueToResultBuf(output, data, type, bytes); - } - - count += 1; - } - - qDebug("QInfo:%p create (tableId, tag) info completed, rows:%d", pQInfo, count); - - } else if (functionId == TSDB_FUNC_COUNT) {// handle the "count(tbname)" query - *(int64_t*) pQuery->sdata[0]->data = num; - - count = 1; - SET_STABLE_QUERY_OVER(pQInfo); - qDebug("QInfo:%p create count(tbname) query, res:%d rows:1", pQInfo, count); - } else { // return only the tags|table name etc. - count = 0; - SSchema tbnameSchema = tGetTableNameColumnSchema(); - - int32_t maxNumOfTables = (int32_t)pQuery->rec.capacity; - if (pQuery->limit.limit >= 0 && pQuery->limit.limit < pQuery->rec.capacity) { - maxNumOfTables = (int32_t)pQuery->limit.limit; - } - - while(pQInfo->tableIndex < num && count < maxNumOfTables) { - int32_t i = pQInfo->tableIndex++; - - // discard current result due to offset - if (pQuery->limit.offset > 0) { - pQuery->limit.offset -= 1; - continue; - } - - SExprInfo* pExprInfo = pQuery->pExpr1; - STableQueryInfo* item = taosArrayGetP(pa, i); - - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.colInfo.flag)) { - continue; - } - - if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - bytes = tbnameSchema.bytes; - type = tbnameSchema.type; - - data = tsdbGetTableName(item->pTable); - dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes; - } else { - type = pExprInfo[j].type; - bytes = pExprInfo[j].bytes; - - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); - dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; - - } - - doSetTagValueToResultBuf(dst, data, type, bytes); - } - count += 1; + if (IS_VAR_DATA_TYPE(type)) { + // Binary data overflows for sort of unknown reasons. Let trim the overflow data + if (varDataTLen(val) > bytes) { + int32_t len = bytes - VARSTR_HEADER_SIZE; // remain available space + memcpy(varDataVal(output), varDataVal(val), len); + varDataSetLen(output, len); + } else { + varDataCopy(output, val); } - - qDebug("QInfo:%p create tag values results completed, rows:%d", pQInfo, count); + } else { + memcpy(output, val, bytes); } - - pQuery->rec.rows = count; - setQueryStatus(pQuery, QUERY_COMPLETED); } static int64_t getQuerySupportBufSize(size_t numOfTables) { @@ -7526,15 +6620,15 @@ static int64_t getQuerySupportBufSize(size_t numOfTables) { int32_t checkForQueryBuf(size_t numOfTables) { int64_t t = getQuerySupportBufSize(numOfTables); - if (tsQueryBufferSize < 0) { + if (tsQueryBufferSizeBytes < 0) { return TSDB_CODE_SUCCESS; - } else if (tsQueryBufferSize > 0) { + } else if (tsQueryBufferSizeBytes > 0) { while(1) { - int64_t s = tsQueryBufferSize; + int64_t s = tsQueryBufferSizeBytes; int64_t remain = s - t; if (remain >= 0) { - if (atomic_val_compare_exchange_64(&tsQueryBufferSize, s, remain) == s) { + if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) { return TSDB_CODE_SUCCESS; } } else { @@ -7548,166 +6642,12 @@ int32_t checkForQueryBuf(size_t numOfTables) { } void releaseQueryBuf(size_t numOfTables) { - if (tsQueryBufferSize <= 0) { + if (tsQueryBufferSizeBytes < 0) { return; } int64_t t = getQuerySupportBufSize(numOfTables); // restore value is not enough buffer available - atomic_add_fetch_64(&tsQueryBufferSize, t); -} - -void* qGetResultRetrieveMsg(qinfo_t qinfo) { - SQInfo* pQInfo = (SQInfo*) qinfo; - assert(pQInfo != NULL); - - return pQInfo->rspContext; -} - -void freeqinfoFn(void *qhandle) { - void** handle = qhandle; - if (handle == NULL || *handle == NULL) { - return; - } - - qKillQuery(*handle); - qDestroyQueryInfo(*handle); -} - -void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool - - char cacheName[128] = {0}; - sprintf(cacheName, "qhandle_%d", vgId); - - SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); - if (pQueryMgmt == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pQueryMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); - pQueryMgmt->closed = false; - pQueryMgmt->vgId = vgId; - - pthread_mutex_init(&pQueryMgmt->lock, NULL); - - qDebug("vgId:%d, open querymgmt success", vgId); - return pQueryMgmt; -} - -static void queryMgmtKillQueryFn(void* handle) { - void** fp = (void**)handle; - qKillQuery(*fp); -} - -void qQueryMgmtNotifyClosed(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt* pQueryMgmt = pQMgmt; - qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); - - pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = true; - pthread_mutex_unlock(&pQueryMgmt->lock); - - taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); -} - -void qQueryMgmtReOpen(void *pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt *pQueryMgmt = pQMgmt; - qDebug("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId); - - pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = false; - pthread_mutex_unlock(&pQueryMgmt->lock); -} - -void qCleanupQueryMgmt(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - SQueryMgmt* pQueryMgmt = pQMgmt; - int32_t vgId = pQueryMgmt->vgId; - - assert(pQueryMgmt->closed); - - SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; - pQueryMgmt->qinfoPool = NULL; - - taosCacheCleanup(pqinfoPool); - pthread_mutex_destroy(&pQueryMgmt->lock); - tfree(pQueryMgmt); - - qDebug("vgId:%d, queryMgmt cleanup completed", vgId); -} - -void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { - if (pMgmt == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - qError("QInfo:%p failed to add qhandle into qMgmt, since qMgmt is closed", (void *)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - pthread_mutex_lock(&pQueryMgmt->lock); - if (pQueryMgmt->closed) { - pthread_mutex_unlock(&pQueryMgmt->lock); - qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } else { - TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE) qInfo; - void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), - (getMaximumIdleDurationSec()*1000)); - pthread_mutex_unlock(&pQueryMgmt->lock); - - return handle; - } -} - -void** qAcquireQInfo(void* pMgmt, uint64_t _key) { - SQueryMgmt *pQueryMgmt = pMgmt; - - if (pQueryMgmt->closed) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - if (pQueryMgmt->qinfoPool == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } - - TSDB_CACHE_PTR_TYPE key = (TSDB_CACHE_PTR_TYPE)_key; - void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &key, sizeof(TSDB_CACHE_PTR_TYPE)); - if (handle == NULL || *handle == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } else { - return handle; - } -} - -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { - SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - return NULL; - } - - taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); - return 0; + atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); } diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index fa3fe285a857cc0d146e25f52710d0ebc0a69193..a73f38528266bfa70790414a2963eb0a8290d7b9 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -12,13 +12,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" #include "qExtbuffer.h" +#include "os.h" +#include "qAggMain.h" #include "queryLog.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" -#include "tsqlfunction.h" #include "tulog.h" #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ @@ -362,20 +362,10 @@ static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, i return (first < second) ? -1 : 1; }; case TSDB_DATA_TYPE_DOUBLE: { - double first = GET_DOUBLE_VAL(f1); - double second = GET_DOUBLE_VAL(f2); - if (first == second) { - return 0; - } - return (first < second) ? -1 : 1; + DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); }; case TSDB_DATA_TYPE_FLOAT: { - float first = GET_FLOAT_VAL(f1); - float second = GET_FLOAT_VAL(f2); - if (first == second) { - return 0; - } - return (first < second) ? -1 : 1; + DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); }; case TSDB_DATA_TYPE_BIGINT: { int64_t first = *(int64_t *)f1; diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index ca1203cb17da52b8ffe638e29913b06bdfcade29..2de1029396fb779798ea7c20da939b2153ca3dd3 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -15,255 +15,27 @@ #include "os.h" +#include "qAggMain.h" #include "taosdef.h" #include "taosmsg.h" -#include "tsqlfunction.h" #include "ttype.h" #include "qFill.h" #include "qExtbuffer.h" #include "queryLog.h" +#include "qExecutor.h" #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) - -// there are no duplicated tags in the SFillTagColInfo list -static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) { - int32_t rowsize = 0; - - int32_t k = 0; - for (int32_t i = 0; i < numOfCols; ++i) { - SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; - pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity); - - if (TSDB_COL_IS_TAG(pColInfo->flag)) { - bool exists = false; - int32_t index = -1; - for (int32_t j = 0; j < k; ++j) { - if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { - exists = true; - index = j; - break; - } - } - - if (!exists) { - SSchema* pSchema = &pFillInfo->pTags[k].col; - pSchema->colId = pColInfo->col.colId; - pSchema->type = pColInfo->col.type; - pSchema->bytes = pColInfo->col.bytes; - - pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); - pColInfo->tagIndex = k; - - k += 1; - } else { - pColInfo->tagIndex = index; - } - } - - rowsize += pColInfo->col.bytes; - } - - assert(k <= pFillInfo->numOfTags); - return rowsize; -} - -SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, - int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, - SFillColInfo* pCol, void* handle) { - if (fillType == TSDB_FILL_NONE) { - return NULL; - } - - SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); - - taosResetFillInfo(pFillInfo, skey); - - pFillInfo->order = order; - pFillInfo->type = fillType; - pFillInfo->pFillCol = pCol; - pFillInfo->numOfTags = numOfTags; - pFillInfo->numOfCols = numOfCols; - pFillInfo->precision = precision; - pFillInfo->alloc = capacity; - pFillInfo->handle = handle; - - pFillInfo->interval.interval = slidingTime; - pFillInfo->interval.intervalUnit = slidingUnit; - pFillInfo->interval.sliding = slidingTime; - pFillInfo->interval.slidingUnit = slidingUnit; - - pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); - if (numOfTags > 0) { - pFillInfo->pTags = calloc(pFillInfo->numOfTags, sizeof(SFillTagColInfo)); - for (int32_t i = 0; i < numOfTags; ++i) { - pFillInfo->pTags[i].col.colId = -2; // TODO - } - } - - pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); - assert(pFillInfo->rowSize > 0); - - return pFillInfo; -} - -void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { - pFillInfo->start = startTimestamp; - pFillInfo->currentKey = startTimestamp; - pFillInfo->index = -1; - pFillInfo->numOfRows = 0; - pFillInfo->numOfCurrent = 0; - pFillInfo->numOfTotal = 0; -} - -void* taosDestroyFillInfo(SFillInfo* pFillInfo) { - if (pFillInfo == NULL) { - return NULL; - } - - tfree(pFillInfo->prevValues); - tfree(pFillInfo->nextValues); - tfree(pFillInfo->pTags); - - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - tfree(pFillInfo->pData[i]); - } - - tfree(pFillInfo->pData); - tfree(pFillInfo->pFillCol); - - tfree(pFillInfo); - return NULL; -} - -void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) { - if (pFillInfo->type == TSDB_FILL_NONE) { - return; - } - - pFillInfo->end = endKey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision); - } - - pFillInfo->index = 0; - pFillInfo->numOfRows = numOfRows; - - // ensure the space - if (pFillInfo->alloc < numOfRows) { - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - char* tmp = realloc(pFillInfo->pData[i], numOfRows*pFillInfo->pFillCol[i].col.bytes); - assert(tmp != NULL); // todo handle error - - memset(tmp, 0, numOfRows*pFillInfo->pFillCol[i].col.bytes); - pFillInfo->pData[i] = tmp; - } - } -} - -// copy the data into source data buffer -void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) { - for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes); - } -} - -void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput) { - assert(pFillInfo->numOfRows == pInput->num); - - for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - - const char* data = pInput->data + pCol->col.offset * pInput->num; - memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes)); - - if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer - SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; - assert (pTag->col.colId == pCol->col.colId); - memcpy(pTag->tagVal, data, pCol->col.bytes); - } - } -} - -int64_t getNumOfResWithFill(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { - int64_t* tsList = (int64_t*) pFillInfo->pData[0]; - - int32_t numOfRows = taosNumOfRemainRows(pFillInfo); - - TSKEY ekey1 = ekey; - if (!FILL_IS_ASC_FILL(pFillInfo)) { - pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); - } - - int64_t numOfRes = -1; - if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. - TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; - numOfRes = taosTimeCountInterval( - lastKey, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->precision); - numOfRes += 1; - assert(numOfRes >= numOfRows); - } else { // reach the end of data - if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || - (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { - return 0; - } - numOfRes = taosTimeCountInterval( - ekey1, - pFillInfo->currentKey, - pFillInfo->interval.sliding, - pFillInfo->interval.slidingUnit, - pFillInfo->precision); - numOfRes += 1; - } - - return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; -} - -int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { - if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) { - return 0; - } - - return pFillInfo->numOfRows - pFillInfo->index; -} - #define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1)))) -int32_t taosGetLinearInterpolationVal(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) { - double v1 = -1; - double v2 = -1; - - GET_TYPED_DATA(v1, double, type, point1->val); - GET_TYPED_DATA(v2, double, type, point2->val); - - double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); - - switch(type) { - case TSDB_DATA_TYPE_TINYINT: *(int8_t*) point->val = (int8_t) r;break; - case TSDB_DATA_TYPE_SMALLINT: *(int16_t*) point->val = (int16_t) r;break; - case TSDB_DATA_TYPE_INT: *(int32_t*) point->val = (int32_t) r;break; - case TSDB_DATA_TYPE_BIGINT: *(int64_t*) point->val = (int64_t) r;break; - case TSDB_DATA_TYPE_DOUBLE: *(double*) point->val = (double) r;break; - case TSDB_DATA_TYPE_FLOAT: *(float*) point->val = (float) r;break; - default: - assert(0); - } - - return TSDB_CODE_SUCCESS; -} - -static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows) { +static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) { SFillColInfo* pCol = &pFillInfo->pFillCol[j]; if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { continue; } - char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, genRows); + char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows); assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags); SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; @@ -273,17 +45,17 @@ static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows } } -static void setNullValueForRow(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfCol, int32_t rowIndex) { +static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. for (int32_t i = 1; i < numOfCol; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; - char* output = elePtrAt(data[i]->data, pCol->col.bytes, rowIndex); + char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex); setNull(output, pCol->col.type, pCol->col.bytes); } } -static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** srcData, int64_t ts, bool outOfBound) { +static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) { char* prev = pFillInfo->prevValues; char* next = pFillInfo->nextValues; @@ -292,13 +64,29 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr // set the primary timestamp column value int32_t index = pFillInfo->numOfCurrent; - char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, index); + char* val = elePtrAt(data[0], TSDB_KEYSIZE, index); *(TSKEY*) val = pFillInfo->currentKey; // set the other values if (pFillInfo->type == TSDB_FILL_PREV) { char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next; - + + if (p != NULL) { + for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + if (TSDB_COL_IS_TAG(pCol->flag)) { + continue; + } + + char* output = elePtrAt(data[i], pCol->col.bytes, index); + assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); + } + } else { // no prev value yet, set the value for NULL + setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); + } + } else if (pFillInfo->type == TSDB_FILL_NEXT) { + char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev; + if (p != NULL) { for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; @@ -306,7 +94,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr continue; } - char* output = elePtrAt(data[i]->data, pCol->col.bytes, index); + char* output = elePtrAt(data[i], pCol->col.bytes, index); assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type); } } else { // no prev value yet, set the value for NULL @@ -323,8 +111,8 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr int16_t type = pCol->col.type; int16_t bytes = pCol->col.bytes; - - char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, index); + + char *val1 = elePtrAt(data[i], pCol->col.bytes, index); if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) { setNull(val1, pCol->col.type, bytes); continue; @@ -333,7 +121,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset}; point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes}; point = (SPoint){.key = pFillInfo->currentKey, .val = val1}; - taosGetLinearInterpolationVal(type, &point1, &point2, &point); + taosGetLinearInterpolationVal(&point, type, &point1, &point2, type); } } else { setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index); @@ -345,7 +133,7 @@ static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** sr continue; } - char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, index); + char* val1 = elePtrAt(data[i], pCol->col.bytes, index); assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type); } } @@ -359,7 +147,7 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) { if (*next != NULL) { return; } - + *next = calloc(1, pFillInfo->rowSize); for (int i = 1; i < pFillInfo->numOfCols; i++) { SFillColInfo* pCol = &pFillInfo->pFillCol[i]; @@ -375,7 +163,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* bu } } -static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t outputRows) { +static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) { pFillInfo->numOfCurrent = 0; char** srcData = pFillInfo->pData; @@ -393,9 +181,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou while (pFillInfo->numOfCurrent < outputRows) { int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index]; + // set the next value for interpolation if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) { - /* set the next value for interpolation */ initBeforeAfterDataBuf(pFillInfo, next); copyCurrentRowIntoBuf(pFillInfo, srcData, *next); } @@ -426,7 +214,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou continue; } - char* output = elePtrAt(data[i]->data, pCol->col.bytes, pFillInfo->numOfCurrent); + char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent); char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index); if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) || @@ -449,7 +237,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent); pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, - pFillInfo->interval.slidingUnit, pFillInfo->precision); + pFillInfo->interval.slidingUnit, pFillInfo->precision); pFillInfo->index += 1; pFillInfo->numOfCurrent += 1; } @@ -468,7 +256,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t ou return pFillInfo->numOfCurrent; } -static int64_t fillExternalResults(SFillInfo* pFillInfo, tFilePage** output, int64_t resultCapacity) { +static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) { /* * These data are generated according to fill strategy, since the current timestamp is out of the time window of * real result set. Note that we need to keep the direct previous result rows, to generated the filled data. @@ -484,15 +272,260 @@ static int64_t fillExternalResults(SFillInfo* pFillInfo, tFilePage** output, int return resultCapacity; } -int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) { +// there are no duplicated tags in the SFillTagColInfo list +static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) { + int32_t rowsize = 0; + + int32_t k = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + SFillColInfo* pColInfo = &pFillInfo->pFillCol[i]; + pFillInfo->pData[i] = NULL; + + if (TSDB_COL_IS_TAG(pColInfo->flag)) { + bool exists = false; + int32_t index = -1; + for (int32_t j = 0; j < k; ++j) { + if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) { + exists = true; + index = j; + break; + } + } + + if (!exists) { + SSchema* pSchema = &pFillInfo->pTags[k].col; + pSchema->colId = pColInfo->col.colId; + pSchema->type = pColInfo->col.type; + pSchema->bytes = pColInfo->col.bytes; + + pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes); + pColInfo->tagIndex = k; + + k += 1; + } else { + pColInfo->tagIndex = index; + } + } + + rowsize += pColInfo->col.bytes; + } + + assert(k <= pFillInfo->numOfTags); + return rowsize; +} + +static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) { + if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) { + return 0; + } + + return pFillInfo->numOfRows - pFillInfo->index; +} + +SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, + int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, + SFillColInfo* pCol, void* handle) { + if (fillType == TSDB_FILL_NONE) { + return NULL; + } + + SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo)); + taosResetFillInfo(pFillInfo, skey); + + pFillInfo->order = order; + pFillInfo->type = fillType; + pFillInfo->pFillCol = pCol; + pFillInfo->numOfTags = numOfTags; + pFillInfo->numOfCols = numOfCols; + pFillInfo->precision = precision; + pFillInfo->alloc = capacity; + pFillInfo->handle = handle; + + pFillInfo->interval.interval = slidingTime; + pFillInfo->interval.intervalUnit = slidingUnit; + pFillInfo->interval.sliding = slidingTime; + pFillInfo->interval.slidingUnit = slidingUnit; + + pFillInfo->pData = malloc(POINTER_BYTES * numOfCols); + if (numOfTags > 0) { + pFillInfo->pTags = calloc(pFillInfo->numOfTags, sizeof(SFillTagColInfo)); + for (int32_t i = 0; i < numOfTags; ++i) { + pFillInfo->pTags[i].col.colId = -2; // TODO + } + } + + pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc); + assert(pFillInfo->rowSize > 0); + + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + pFillInfo->pData[i] = malloc(pFillInfo->pFillCol[i].col.bytes * pFillInfo->alloc); + } + + return pFillInfo; +} + +void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { + pFillInfo->start = startTimestamp; + pFillInfo->currentKey = startTimestamp; + pFillInfo->index = -1; + pFillInfo->numOfRows = 0; + pFillInfo->numOfCurrent = 0; + pFillInfo->numOfTotal = 0; +} + +void* taosDestroyFillInfo(SFillInfo* pFillInfo) { + if (pFillInfo == NULL) { + return NULL; + } + + tfree(pFillInfo->prevValues); + tfree(pFillInfo->nextValues); + + for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) { + tfree(pFillInfo->pTags[i].tagVal); + } + + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + tfree(pFillInfo->pData[i]); + } + + tfree(pFillInfo->pTags); + + tfree(pFillInfo->pData); + tfree(pFillInfo->pFillCol); + + tfree(pFillInfo); + return NULL; +} + +void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) { + if (pFillInfo->type == TSDB_FILL_NONE) { + return; + } + + pFillInfo->end = endKey; + if (!FILL_IS_ASC_FILL(pFillInfo)) { + pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision); + } + + pFillInfo->index = 0; + pFillInfo->numOfRows = numOfRows; + + // ensure the space + if (pFillInfo->alloc < numOfRows) { + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + char* tmp = realloc(pFillInfo->pData[i], numOfRows*pFillInfo->pFillCol[i].col.bytes); + assert(tmp != NULL); // todo handle error + + memset(tmp, 0, numOfRows*pFillInfo->pFillCol[i].col.bytes); + pFillInfo->pData[i] = tmp; + } + } +} + +void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) { + for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i); +// pFillInfo->pData[i] = pColData->pData; + if (pInput->info.rows > pFillInfo->alloc) { + char* t = realloc(pFillInfo->pData[i], pColData->info.bytes * pInput->info.rows); + assert(t != NULL); + + pFillInfo->pData[i] = t; + pFillInfo->alloc = pInput->info.rows; + } + + memcpy(pFillInfo->pData[i], pColData->pData, pColData->info.bytes * pInput->info.rows); + } +} + +void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput) { + assert(pFillInfo->numOfRows == pInput->num); + + for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { + SFillColInfo* pCol = &pFillInfo->pFillCol[i]; + + const char* data = pInput->data + pCol->col.offset * pInput->num; + if (pInput->num > pFillInfo->alloc) { + char* t = realloc(pFillInfo->pData[i], (size_t)(pCol->col.bytes * pInput->num)); + assert(t != NULL); + + pFillInfo->pData[i] = t; + pFillInfo->alloc = (int32_t)pInput->num; + } + + memcpy(pFillInfo->pData[i], data, (size_t)(pCol->col.bytes * pInput->num)); + + if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer + SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex]; + assert (pTag->col.colId == pCol->col.colId); + memcpy(pTag->tagVal, data, pCol->col.bytes); // TODO not memcpy?? + } + } +} + +bool taosFillHasMoreResults(SFillInfo* pFillInfo) { + return taosNumOfRemainRows(pFillInfo) > 0; +} + +int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) { + int64_t* tsList = (int64_t*) pFillInfo->pData[0]; + + int32_t numOfRows = taosNumOfRemainRows(pFillInfo); + + TSKEY ekey1 = ekey; + if (!FILL_IS_ASC_FILL(pFillInfo)) { + pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); + } + + int64_t numOfRes = -1; + if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set. + TSKEY lastKey = tsList[pFillInfo->numOfRows - 1]; + numOfRes = taosTimeCountInterval( + lastKey, + pFillInfo->currentKey, + pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, + pFillInfo->precision); + numOfRes += 1; + assert(numOfRes >= numOfRows); + } else { // reach the end of data + if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || + (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { + return 0; + } + numOfRes = taosTimeCountInterval( + ekey1, + pFillInfo->currentKey, + pFillInfo->interval.sliding, + pFillInfo->interval.slidingUnit, + pFillInfo->precision); + numOfRes += 1; + } + + return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes; +} + +int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) { + double v1 = -1, v2 = -1; + GET_TYPED_DATA(v1, double, inputType, point1->val); + GET_TYPED_DATA(v2, double, inputType, point2->val); + + double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); + SET_TYPED_DATA(point->val, outputType, r); + + return TSDB_CODE_SUCCESS; +} + +int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) { int32_t remain = taosNumOfRemainRows(pFillInfo); - int64_t numOfRes = getNumOfResWithFill(pFillInfo, pFillInfo->end, capacity); + int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); assert(numOfRes <= capacity); // no data existed for fill operation now, append result according to the fill strategy if (remain == 0) { - fillExternalResults(pFillInfo, output, numOfRes); + appendFilledResult(pFillInfo, output, numOfRes); } else { fillResultImpl(pFillInfo, output, (int32_t) numOfRes); assert(numOfRes == pFillInfo->numOfCurrent); diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index bdc071060c0e1f5beecf18c239a684f8e77714e7..ae25a752347edc4f7c53aade845716d637f9edd7 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -184,7 +184,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { histogramCreateBin(*pHisto, idx, val); } #else - tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); SHistBin* entry = calloc(1, sizeof(SHistBin)); entry->val = val; @@ -217,7 +217,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { } tSkipListKey kx = - tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize); pLast = tSkipListGetOne((*pHisto)->pList, &kx); } } else { diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c deleted file mode 100644 index d311cb35579f78490db540c419c81552607808f9..0000000000000000000000000000000000000000 --- a/src/query/src/qParserImpl.c +++ /dev/null @@ -1,861 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "qSqlparser.h" -#include "taosdef.h" -#include "taosmsg.h" -#include "tcmdtype.h" -#include "tstoken.h" -#include "tstrbuild.h" -#include "ttokendef.h" -#include "tutil.h" - -SSqlInfo qSQLParse(const char *pStr) { - void *pParser = ParseAlloc(malloc); - - SSqlInfo sqlInfo = {0}; - sqlInfo.valid = true; - - int32_t i = 0; - while (1) { - SStrToken t0 = {0}; - - if (pStr[i] == 0) { - Parse(pParser, 0, t0, &sqlInfo); - goto abort_parse; - } - - t0.n = tSQLGetToken((char *)&pStr[i], &t0.type); - t0.z = (char *)(pStr + i); - i += t0.n; - - switch (t0.type) { - case TK_SPACE: - case TK_COMMENT: { - break; - } - case TK_SEMI: { - Parse(pParser, 0, t0, &sqlInfo); - goto abort_parse; - } - - case TK_QUESTION: - case TK_ILLEGAL: { - snprintf(sqlInfo.pzErrMsg, tListLen(sqlInfo.pzErrMsg), "unrecognized token: \"%s\"", t0.z); - sqlInfo.valid = false; - goto abort_parse; - } - default: - Parse(pParser, t0.type, t0, &sqlInfo); - if (sqlInfo.valid == false) { - goto abort_parse; - } - } - } - -abort_parse: - ParseFree(pParser, free); - return sqlInfo; -} - -tSQLExprList *tSqlExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { - if (pList == NULL) { - pList = calloc(1, sizeof(tSQLExprList)); - } - - if (pList->nAlloc <= pList->nExpr) { - pList->nAlloc = (pList->nAlloc << 1u) + 4; - pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); - if (pList->a == 0) { - pList->nExpr = pList->nAlloc = 0; - return pList; - } - } - assert(pList->a != 0); - - if (pNode || pToken) { - struct tSqlExprItem *pItem = &pList->a[pList->nExpr++]; - memset(pItem, 0, sizeof(*pItem)); - pItem->pNode = pNode; - if (pToken) { // set the as clause - pItem->aliasName = malloc(pToken->n + 1); - strncpy(pItem->aliasName, pToken->z, pToken->n); - pItem->aliasName[pToken->n] = 0; - - strdequote(pItem->aliasName); - } - } - return pList; -} - -void tSqlExprListDestroy(tSQLExprList *pList) { - if (pList == NULL) return; - - for (int32_t i = 0; i < pList->nExpr; ++i) { - if (pList->a[i].aliasName != NULL) { - free(pList->a[i].aliasName); - } - tSqlExprDestroy(pList->a[i].pNode); - } - - free(pList->a); - free(pList); -} - -tSQLExpr *tSqlExprIdValueCreate(SStrToken *pToken, int32_t optrType) { - tSQLExpr *pSqlExpr = calloc(1, sizeof(tSQLExpr)); - - if (pToken != NULL) { - pSqlExpr->token = *pToken; - } - - if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - toTSDBType(pToken->type); - - tVariantCreate(&pSqlExpr->val, pToken); - pSqlExpr->nSQLOptr = optrType; - } else if (optrType == TK_NOW) { - // use microsecond by default - pSqlExpr->val.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pSqlExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond - } else if (optrType == TK_VARIABLE) { - int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->val.i64); - if (ret != TSDB_CODE_SUCCESS) { - terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - } - - pSqlExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pSqlExpr->nSQLOptr = TK_TIMESTAMP; - } else { // it must be the column name (tk_id) if it is not the number - assert(optrType == TK_ID || optrType == TK_ALL); - if (pToken != NULL) { - pSqlExpr->colInfo = *pToken; - } - - pSqlExpr->nSQLOptr = optrType; - } - - return pSqlExpr; -} - -/* - * pList is the parameters for function with id(optType) - * function name is denoted by pFunctionToken - */ -tSQLExpr *tSqlExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { - if (pFuncToken == NULL) return NULL; - - tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - pExpr->nSQLOptr = optType; - pExpr->pParam = pList; - - int32_t len = (int32_t)((endToken->z + endToken->n) - pFuncToken->z); - pExpr->operand.z = pFuncToken->z; - - pExpr->operand.n = len; // raw field name - pExpr->operand.type = pFuncToken->type; - - pExpr->token = pExpr->operand; - return pExpr; -} - -/* - * create binary expression in this procedure - * if the expr is arithmetic, calculate the result and set it to tSQLExpr Object - */ -tSQLExpr *tSqlExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { - tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - - if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { - char* endPos = pRight->token.z + pRight->token.n; - pExpr->token.z = pLeft->token.z; - pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); - pExpr->token.type = pLeft->token.type; - } - - if ((pLeft != NULL && pRight != NULL) && - (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) { - /* - * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond - * Otherwise, the time precision is adaptive, determined by the time precision from databases. - */ - if ((pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_INTEGER) || - (pLeft->nSQLOptr == TK_TIMESTAMP && pRight->nSQLOptr == TK_TIMESTAMP)) { - pExpr->val.nType = TSDB_DATA_TYPE_BIGINT; - pExpr->nSQLOptr = pLeft->nSQLOptr; - - switch (optrType) { - case TK_PLUS: { - pExpr->val.i64 = pLeft->val.i64 + pRight->val.i64; - break; - } - case TK_MINUS: { - pExpr->val.i64 = pLeft->val.i64 - pRight->val.i64; - break; - } - case TK_STAR: { - pExpr->val.i64 = pLeft->val.i64 * pRight->val.i64; - break; - } - case TK_DIVIDE: { - pExpr->nSQLOptr = TK_FLOAT; - pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->val.dKey = (double)pLeft->val.i64 / pRight->val.i64; - break; - } - case TK_REM: { - pExpr->val.i64 = pLeft->val.i64 % pRight->val.i64; - break; - } - } - - tSqlExprDestroy(pLeft); - tSqlExprDestroy(pRight); - - } else if ((pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_INTEGER) || (pLeft->nSQLOptr == TK_INTEGER && pRight->nSQLOptr == TK_FLOAT) || - (pLeft->nSQLOptr == TK_FLOAT && pRight->nSQLOptr == TK_FLOAT)) { - pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE; - pExpr->nSQLOptr = TK_FLOAT; - - double left = (pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->val.dKey : pLeft->val.i64; - double right = (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->val.dKey : pRight->val.i64; - - switch (optrType) { - case TK_PLUS: { - pExpr->val.dKey = left + right; - break; - } - case TK_MINUS: { - pExpr->val.dKey = left - right; - break; - } - case TK_STAR: { - pExpr->val.dKey = left * right; - break; - } - case TK_DIVIDE: { - pExpr->val.dKey = left / right; - break; - } - case TK_REM: { - pExpr->val.dKey = left - ((int64_t)(left / right)) * right; - break; - } - } - - tSqlExprDestroy(pLeft); - tSqlExprDestroy(pRight); - - } else { - pExpr->nSQLOptr = optrType; - pExpr->pLeft = pLeft; - pExpr->pRight = pRight; - } - } else if (optrType == TK_IN) { - pExpr->nSQLOptr = optrType; - pExpr->pLeft = pLeft; - - tSQLExpr *pRSub = calloc(1, sizeof(tSQLExpr)); - pRSub->nSQLOptr = TK_SET; // TODO refactor ..... - pRSub->pParam = (tSQLExprList *)pRight; - - pExpr->pRight = pRSub; - } else { - pExpr->nSQLOptr = optrType; - pExpr->pLeft = pLeft; - - if (pRight == NULL) { - pRight = calloc(1, sizeof(tSQLExpr)); - } - - pExpr->pRight = pRight; - } - - return pExpr; -} - -void tSqlExprNodeDestroy(tSQLExpr *pExpr) { - if (pExpr == NULL) { - return; - } - - if (pExpr->nSQLOptr == TK_STRING) { - tVariantDestroy(&pExpr->val); - } - - tSqlExprListDestroy(pExpr->pParam); - - free(pExpr); -} - -void tSqlExprDestroy(tSQLExpr *pExpr) { - if (pExpr == NULL) { - return; - } - - tSqlExprDestroy(pExpr->pLeft); - tSqlExprDestroy(pExpr->pRight); - - tSqlExprNodeDestroy(pExpr); -} - -SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tVariantListItem)); - } - - if (pToken) { - tVariantListItem item; - tVariantCreate(&item.pVar, pToken); - item.sortOrder = order; - - taosArrayPush(pList, &item); - } - - return pList; -} - -SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) { - if (pList == NULL) { - pList = taosArrayInit(4, sizeof(tVariantListItem)); - } - - if (pVar == NULL) { - return pList; - } - - /* - * Here we do not employ the assign function, since we need the pz attribute of structure - * , which is the point to char string, to free it! - * - * Otherwise, the original pointer may be lost, which causes memory leak. - */ - tVariantListItem item; - item.pVar = *pVar; - item.sortOrder = sortOrder; - - taosArrayPush(pList, &item); - return pList; -} - -SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) { - if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) { - return tVariantListAppend(NULL, pVar, sortOrder); - } - - tVariantListItem item; - - item.pVar = *pVar; - item.sortOrder = sortOrder; - - taosArrayInsert(pList, index, &item); - return pList; -} - -void setDbName(SStrToken *pCpxName, SStrToken *pDb) { - pCpxName->type = pDb->type; - pCpxName->z = pDb->z; - pCpxName->n = pDb->n; -} - -void tSqlSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { - int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); - - // truncate the column name - if ((int32_t)pName->n >= maxLen) { - pName->n = maxLen - 1; - } - - strncpy(pField->name, pName->z, pName->n); - pField->name[pName->n] = 0; - - pField->type = pType->type; - if(!isValidDataType(pField->type)){ - pField->bytes = 0; - } else { - pField->bytes = pType->bytes; - } - -} - -void tSqlSetColumnType(TAOS_FIELD *pField, SStrToken *type) { - // set the field type invalid - pField->type = -1; - pField->name[0] = 0; - - int32_t i = 0; - while (i < tListLen(tDataTypeDesc)) { - if ((type->n == tDataTypeDesc[i].nameLen) && - (strncasecmp(type->z, tDataTypeDesc[i].aName, tDataTypeDesc[i].nameLen) == 0)) { - break; - } - - i += 1; - } - - if (i == tListLen(tDataTypeDesc)) { - return; - } - - pField->type = i; - pField->bytes = tDataTypeDesc[i].nSize; - - if (i == TSDB_DATA_TYPE_NCHAR) { - /* - * for nchar, the TOKENTYPE is the number of character, so the length is the - * number of bytes in UCS-4 format, which is 4 times larger than the number of characters - */ - if (type->type == 0) { - pField->bytes = 0; - } else { - int32_t bytes = -(int32_t)(type->type); - if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - // we have to postpone reporting the error because it cannot be done here - // as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow - bytes = TSDB_MAX_NCHAR_LEN + 1; - } else { - bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } - pField->bytes = (int16_t)bytes; - } - } else if (i == TSDB_DATA_TYPE_BINARY) { - /* for binary, the TOKENTYPE is the length of binary */ - if (type->type == 0) { - pField->bytes = 0; - } else { - int32_t bytes = -(int32_t)(type->type); - if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - // refer comment for NCHAR above - bytes = TSDB_MAX_BINARY_LEN + 1; - } else { - bytes += VARSTR_HEADER_SIZE; - } - - pField->bytes = (int16_t)bytes; - } - } -} - -/* - * extract the select info out of sql string - */ -SQuerySQL *tSetQuerySqlElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere, - SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, - SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { - assert(pSelection != NULL); - - SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL)); - pQuery->selectToken = *pSelectToken; - pQuery->selectToken.n = (uint32_t)strlen(pQuery->selectToken.z); // all later sql string are belonged to the stream sql - - pQuery->pSelection = pSelection; - pQuery->from = pFrom; - pQuery->pGroupby = pGroupby; - pQuery->pSortOrder = pSortOrder; - pQuery->pWhere = pWhere; - - if (pLimit != NULL) { - pQuery->limit = *pLimit; - } - - if (pGLimit != NULL) { - pQuery->slimit = *pGLimit; - } - - if (pInterval != NULL) { - pQuery->interval = pInterval->interval; - pQuery->offset = pInterval->offset; - } - - if (pSliding != NULL) { - pQuery->sliding = *pSliding; - } - - pQuery->fillType = pFill; - return pQuery; -} - -static void freeVariant(void *pItem) { - tVariantListItem* p = (tVariantListItem*) pItem; - tVariantDestroy(&p->pVar); -} - -void freeCreateTableInfo(void* p) { - SCreatedTableInfo* pInfo = (SCreatedTableInfo*) p; - taosArrayDestroyEx(pInfo->pTagVals, freeVariant); - tfree(pInfo->fullname); - tfree(pInfo->tagdata.data); -} - -void doDestroyQuerySql(SQuerySQL *pQuerySql) { - if (pQuerySql == NULL) { - return; - } - - tSqlExprListDestroy(pQuerySql->pSelection); - - pQuerySql->pSelection = NULL; - - tSqlExprDestroy(pQuerySql->pWhere); - pQuerySql->pWhere = NULL; - - taosArrayDestroyEx(pQuerySql->pSortOrder, freeVariant); - pQuerySql->pSortOrder = NULL; - - taosArrayDestroyEx(pQuerySql->pGroupby, freeVariant); - pQuerySql->pGroupby = NULL; - - taosArrayDestroyEx(pQuerySql->from, freeVariant); - pQuerySql->from = NULL; - - taosArrayDestroyEx(pQuerySql->fillType, freeVariant); - pQuerySql->fillType = NULL; - - free(pQuerySql); -} - -void destroyAllSelectClause(SSubclauseInfo *pClause) { - if (pClause == NULL || pClause->numOfClause == 0) { - return; - } - - for(int32_t i = 0; i < pClause->numOfClause; ++i) { - SQuerySQL *pQuerySql = pClause->pClause[i]; - doDestroyQuerySql(pQuerySql); - } - - tfree(pClause->pClause); -} - -SCreateTableSQL *tSetCreateSqlElems(SArray *pCols, SArray *pTags, SQuerySQL *pSelect, int32_t type) { - SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL)); - - switch (type) { - case TSQL_CREATE_TABLE: { - pCreate->colInfo.pColumns = pCols; - assert(pTags == NULL); - break; - } - case TSQL_CREATE_STABLE: { - pCreate->colInfo.pColumns = pCols; - pCreate->colInfo.pTagColumns = pTags; - assert(pTags != NULL && pCols != NULL); - break; - } - case TSQL_CREATE_STREAM: { - pCreate->pSelect = pSelect; - break; - } - - case TSQL_CREATE_TABLE_FROM_STABLE: { - assert(0); - } - - default: - assert(false); - } - - pCreate->type = type; - return pCreate; -} - -SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists) { - SCreatedTableInfo info; - memset(&info, 0, sizeof(SCreatedTableInfo)); - - info.name = *pToken; - info.pTagVals = pTagVals; - info.stableName = *pTableName; - info.igExist = (igExists->n > 0)? 1:0; - - return info; -} - -SAlterTableSQL *tAlterTableSqlElems(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type) { - SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL)); - - pAlterTable->name = *pTableName; - pAlterTable->type = type; - - if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { - pAlterTable->pAddColumns = pCols; - assert(pVals == NULL); - } else { - /* - * ALTER_TABLE_TAGS_CHG, ALTER_TABLE_TAGS_SET, ALTER_TABLE_TAGS_DROP, - * ALTER_TABLE_DROP_COLUMN - */ - pAlterTable->varList = pVals; - assert(pCols == NULL); - } - - return pAlterTable; -} - -void* destroyCreateTableSql(SCreateTableSQL* pCreate) { - doDestroyQuerySql(pCreate->pSelect); - - taosArrayDestroy(pCreate->colInfo.pColumns); - taosArrayDestroy(pCreate->colInfo.pTagColumns); - - taosArrayDestroyEx(pCreate->childTableInfo, freeCreateTableInfo); - tfree(pCreate); - - return NULL; -} - -void SqlInfoDestroy(SSqlInfo *pInfo) { - if (pInfo == NULL) return; - - if (pInfo->type == TSDB_SQL_SELECT) { - destroyAllSelectClause(&pInfo->subclauseInfo); - } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) { - pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo); - } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { - taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant); - taosArrayDestroy(pInfo->pAlterInfo->pAddColumns); - tfree(pInfo->pAlterInfo->tagData.data); - tfree(pInfo->pAlterInfo); - } else { - if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) { - free(pInfo->pDCLInfo->a); - } - - if (pInfo->pDCLInfo != NULL && pInfo->type == TSDB_SQL_CREATE_DB) { - taosArrayDestroyEx(pInfo->pDCLInfo->dbOpt.keep, freeVariant); - } - - tfree(pInfo->pDCLInfo); - } -} - -SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { - if (pSubclause == NULL) { - pSubclause = calloc(1, sizeof(SSubclauseInfo)); - } - - int32_t newSize = pSubclause->numOfClause + 1; - char* tmp = realloc(pSubclause->pClause, newSize * POINTER_BYTES); - if (tmp == NULL) { - return pSubclause; - } - - pSubclause->pClause = (SQuerySQL**) tmp; - - pSubclause->pClause[newSize - 1] = pSqlExprInfo; - pSubclause->numOfClause++; - - return pSubclause; -} - -SSqlInfo*setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type) { - pInfo->type = type; - - if (type == TSDB_SQL_SELECT) { - pInfo->subclauseInfo = *(SSubclauseInfo*) pSqlExprInfo; - free(pSqlExprInfo); - } else { - pInfo->pCreateTableInfo = pSqlExprInfo; - } - - if (pTableName != NULL) { - pInfo->pCreateTableInfo->name = *pTableName; - } - - return pInfo; -} - -SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) { - char* tmp = realloc(pQueryInfo->pClause, (pQueryInfo->numOfClause + 1) * POINTER_BYTES); - if (tmp == NULL) { // out of memory - return pQueryInfo; - } - - pQueryInfo->pClause = (SQuerySQL**) tmp; - pQueryInfo->pClause[pQueryInfo->numOfClause++] = pSubclause; - - return pQueryInfo; -} - -void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists) { - pInfo->pCreateTableInfo->name = *pTableNameToken; - pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); -} - -void tTokenListBuyMoreSpace(tDCLSQL *pTokenList) { - if (pTokenList->nAlloc <= pTokenList->nTokens) { // - pTokenList->nAlloc = (pTokenList->nAlloc << 1u) + 4; - pTokenList->a = realloc(pTokenList->a, pTokenList->nAlloc * sizeof(pTokenList->a[0])); - if (pTokenList->a == 0) { - pTokenList->nTokens = pTokenList->nAlloc = 0; - } - } -} - -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken) { - if (pToken == NULL) return NULL; - - if (pTokenList == NULL) pTokenList = calloc(1, sizeof(tDCLSQL)); - - tTokenListBuyMoreSpace(pTokenList); - pTokenList->a[pTokenList->nTokens++] = *pToken; - - return pTokenList; -} - -void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { - pInfo->type = type; - - if (nParam == 0) return; - if (pInfo->pDCLInfo == NULL) pInfo->pDCLInfo = (tDCLSQL *)calloc(1, sizeof(tDCLSQL)); - - va_list va; - va_start(va, nParam); - - while (nParam-- > 0) { - SStrToken *pToken = va_arg(va, SStrToken *); - pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); - } - va_end(va); -} - -void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { - pInfo->type = type; - pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); - pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1); -} - -void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); - } - - pInfo->type = TSDB_SQL_SHOW; - - SShowInfo* pShowInfo = &pInfo->pDCLInfo->showOpt; - pShowInfo->showType = type; - - if (prefix != NULL && prefix->type != 0) { - pShowInfo->prefix = *prefix; - } else { - pShowInfo->prefix.type = 0; - } - - if (pPatterns != NULL && pPatterns->type != 0) { - pShowInfo->pattern = *pPatterns; - } else { - pShowInfo->pattern.type = 0; - } -} - -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists) { - pInfo->type = type; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); - } - - pInfo->pDCLInfo->dbOpt = *pDB; - pInfo->pDCLInfo->dbOpt.dbname = *pToken; - pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} -} - -void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { - pInfo->type = type; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); - } - - pInfo->pDCLInfo->acctOpt = *pAcctInfo; - - assert(pName != NULL); - pInfo->pDCLInfo->user.user = *pName; - - if (pPwd != NULL) { - pInfo->pDCLInfo->user.passwd = *pPwd; - } -} - -void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { - pInfo->type = TSDB_SQL_CREATE_USER; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); - } - - assert(pName != NULL && pPasswd != NULL); - - pInfo->pDCLInfo->user.user = *pName; - pInfo->pDCLInfo->user.passwd = *pPasswd; -} - -void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { - pInfo->type = TSDB_SQL_ALTER_USER; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); - } - - assert(pName != NULL); - - SUserInfo* pUser = &pInfo->pDCLInfo->user; - pUser->type = type; - pUser->user = *pName; - - if (pPwd != NULL) { - pUser->passwd = *pPwd; - } else { - pUser->passwd.type = TSDB_DATA_TYPE_NULL; - } - - if (pPrivilege != NULL) { - pUser->privilege = *pPrivilege; - } else { - pUser->privilege.type = TSDB_DATA_TYPE_NULL; - } -} - -void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { - pInfo->type = type; - if (pInfo->pDCLInfo == NULL) { - pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); - } - - assert(ip != NULL); - - pInfo->pDCLInfo->ip = *ip; -} - -void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { - pDBInfo->compressionLevel = -1; - - pDBInfo->walLevel = -1; - pDBInfo->fsyncPeriod = -1; - pDBInfo->commitTime = -1; - pDBInfo->maxTablesPerVnode = -1; - - pDBInfo->cacheBlockSize = -1; - pDBInfo->numOfBlocks = -1; - pDBInfo->maxRowsPerBlock = -1; - pDBInfo->minRowsPerBlock = -1; - pDBInfo->daysPerFile = -1; - - pDBInfo->replica = -1; - pDBInfo->quorum = -1; - pDBInfo->keep = NULL; - - pDBInfo->update = -1; - pDBInfo->cachelast = 0; - memset(&pDBInfo->precision, 0, sizeof(SStrToken)); -} diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 3e4891092a9696f871f2295555156730bd7b224a..e3326cc26bc4216d131211473e807a9845d49133 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -115,6 +115,11 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { GET_TYPED_DATA(v, int64_t, pBucket->type, value); int32_t index = -1; + + if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) { + return index; + } + // divide the value range into 1024 buckets uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal; if (span < pBucket->numOfSlots) { @@ -128,7 +133,7 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { } } - assert(v >= pBucket->range.i64MinVal && v <= pBucket->range.i64MaxVal && index >= 0 && index < pBucket->numOfSlots); + assert(index >= 0 && index < pBucket->numOfSlots); return index; } @@ -137,6 +142,11 @@ int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { GET_TYPED_DATA(v, uint64_t, pBucket->type, value); int32_t index = -1; + + if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) { + return index; + } + // divide the value range into 1024 buckets uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal; if (span < pBucket->numOfSlots) { @@ -150,7 +160,7 @@ int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) { } } - assert(v >= pBucket->range.u64MinVal && v <= pBucket->range.i64MaxVal && index >= 0 && index < pBucket->numOfSlots); + assert(index >= 0 && index < pBucket->numOfSlots); return index; } @@ -164,6 +174,10 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { int32_t index = -1; + if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) { + return index; + } + // divide a range of [dMinVal, dMaxVal] into 1024 buckets double span = pBucket->range.dMaxVal - pBucket->range.dMinVal; if (span < pBucket->numOfSlots) { @@ -177,7 +191,7 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) { } } - assert(v >= pBucket->range.dMinVal && v <= pBucket->range.dMaxVal && index >= 0 && index < pBucket->numOfSlots); + assert(index >= 0 && index < pBucket->numOfSlots); return index; } @@ -240,7 +254,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bytes, pBucket->bufPageSize, pBucket->bufPageSize * 512, NULL); + int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1); if (ret != TSDB_CODE_SUCCESS) { tMemBucketDestroy(pBucket); return NULL; @@ -309,9 +323,13 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { int32_t bytes = pBucket->bytes; for (int32_t i = 0; i < size; ++i) { char *d = (char *) data + i * bytes; - count += 1; int32_t index = (pBucket->hashFunc)(pBucket, d); + if (index < 0) { + continue; + } + + count += 1; tMemBucketSlot *pSlot = &pBucket->pSlots[index]; tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type); diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index d45e76c2fd07f37afa0ffad0f9db65d5e1948c43..c5dd6b3cac66cfa58f74e4ff0897761bd1704ce7 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -9,8 +9,7 @@ #define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize, - int32_t inMemBufSize, const void* handle) { +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId) { *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); SDiskbasedResultBuf* pResBuf = *pResultBuf; @@ -25,13 +24,12 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro pResBuf->allocateId = -1; pResBuf->comp = true; pResBuf->file = NULL; - pResBuf->handle = handle; + pResBuf->qId = qId; pResBuf->fileSize = 0; // at least more than 2 pages must be in memory assert(inMemBufSize >= pagesize * 2); - pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize; pResBuf->lruList = tdListNew(POINTER_BYTES); // init id hash table @@ -45,7 +43,7 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); - qDebug("QInfo:%p create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", handle, pResBuf->pageSize, + qDebug("QInfo:%"PRIu64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize, pResBuf->inMemPages, pResBuf->path); return TSDB_CODE_SUCCESS; @@ -387,8 +385,6 @@ void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) { pResultBuf->statis.releasePages += 1; } -size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } - size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); } size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; } @@ -410,13 +406,13 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb", - pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, + qDebug("QInfo:%"PRIu64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb", + pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, pResultBuf->fileSize/1024.0); fclose(pResultBuf->file); } else { - qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, no file created", pResultBuf->handle, + qDebug("QInfo:%"PRIu64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId, pResultBuf->totalBufSize/1024.0); } diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c new file mode 100644 index 0000000000000000000000000000000000000000..dc1be4fe01a532f9edfa6bfe146adb5d9cce1bf1 --- /dev/null +++ b/src/query/src/qSqlParser.c @@ -0,0 +1,1071 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "qSqlparser.h" +#include "taosdef.h" +#include "taosmsg.h" +#include "tcmdtype.h" +#include "tstoken.h" +#include "tstrbuild.h" +#include "ttokendef.h" +#include "tutil.h" + +SSqlInfo qSqlParse(const char *pStr) { + void *pParser = ParseAlloc(malloc); + + SSqlInfo sqlInfo = {0}; + sqlInfo.valid = true; + + int32_t i = 0; + while (1) { + SStrToken t0 = {0}; + + if (pStr[i] == 0) { + Parse(pParser, 0, t0, &sqlInfo); + goto abort_parse; + } + + t0.n = tSQLGetToken((char *)&pStr[i], &t0.type); + t0.z = (char *)(pStr + i); + i += t0.n; + + switch (t0.type) { + case TK_SPACE: + case TK_COMMENT: { + break; + } + case TK_SEMI: { + Parse(pParser, 0, t0, &sqlInfo); + goto abort_parse; + } + + case TK_QUESTION: + case TK_ILLEGAL: { + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unrecognized token: \"%s\"", t0.z); + sqlInfo.valid = false; + goto abort_parse; + } + + case TK_HEX: + case TK_OCT: + case TK_BIN:{ + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unsupported token: \"%s\"", t0.z); + sqlInfo.valid = false; + goto abort_parse; + } + + default: + Parse(pParser, t0.type, t0, &sqlInfo); + if (sqlInfo.valid == false) { + goto abort_parse; + } + } + } + +abort_parse: + ParseFree(pParser, free); + return sqlInfo; +} + +SArray *tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SStrToken *pDistinct, SStrToken *pToken) { + if (pList == NULL) { + pList = taosArrayInit(4, sizeof(tSqlExprItem)); + } + + if (pNode || pToken) { + struct tSqlExprItem item = {0}; + + item.pNode = pNode; + item.distinct = (pDistinct != NULL); + + if (pToken) { // set the as clause + item.aliasName = malloc(pToken->n + 1); + strncpy(item.aliasName, pToken->z, pToken->n); + item.aliasName[pToken->n] = 0; + + strdequote(item.aliasName); + } + + taosArrayPush(pList, &item); + } + + return pList; +} + +static void freeExprElem(void* item) { + tSqlExprItem* exprItem = item; + + tfree(exprItem->aliasName); + tSqlExprDestroy(exprItem->pNode); +} + +void tSqlExprListDestroy(SArray *pList) { + if (pList == NULL) { + return; + } + + taosArrayDestroyEx(pList, freeExprElem); +} + +tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { + tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr)); + + if (pToken != NULL) { + pSqlExpr->token = *pToken; + } + + if (optrType == TK_NULL) { + pToken->type = TSDB_DATA_TYPE_NULL; + tVariantCreate(&pSqlExpr->value, pToken); + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_VALUE; + } else if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { + toTSDBType(pToken->type); + + tVariantCreate(&pSqlExpr->value, pToken); + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_VALUE; + } else if (optrType == TK_NOW) { + // use microsecond by default + pSqlExpr->value.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->tokenId = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSqlExpr->type = SQL_NODE_VALUE; + } else if (optrType == TK_VARIABLE) { + int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64); + if (ret != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } + + pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->tokenId = TK_TIMESTAMP; + pSqlExpr->type = SQL_NODE_VALUE; + } else { + // Here it must be the column name (tk_id) if it is not a number or string. + assert(optrType == TK_ID || optrType == TK_ALL); + if (pToken != NULL) { + pSqlExpr->colInfo = *pToken; + } + + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_TABLE_COLUMN; + } + + return pSqlExpr; +} + +/* + * pList is the parameters for function with id(optType) + * function name is denoted by pFunctionToken + */ +tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { + if (pFuncToken == NULL) { + return NULL; + } + + tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); + pExpr->tokenId = optType; + pExpr->type = SQL_NODE_SQLFUNCTION; + pExpr->pParam = pParam; + + int32_t len = (int32_t)((endToken->z + endToken->n) - pFuncToken->z); + pExpr->operand = (*pFuncToken); + + pExpr->token.n = len; + pExpr->token.z = pFuncToken->z; + pExpr->token.type = pFuncToken->type; + + return pExpr; +} + +/* + * create binary expression in this procedure + * if the expr is arithmetic, calculate the result and set it to tSqlExpr Object + */ +tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) { + tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); + + pExpr->type = SQL_NODE_EXPR; + if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { + char* endPos = pRight->token.z + pRight->token.n; + pExpr->token.z = pLeft->token.z; + pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); + pExpr->token.type = pLeft->token.type; + } + + if ((pLeft != NULL && pRight != NULL) && + (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) { + /* + * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond + * Otherwise, the time precision is adaptive, determined by the time precision from databases. + */ + if ((pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_INTEGER) || + (pLeft->tokenId == TK_TIMESTAMP && pRight->tokenId == TK_TIMESTAMP)) { + pExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pExpr->tokenId = pLeft->tokenId; + pExpr->type = SQL_NODE_VALUE; + + switch (optrType) { + case TK_PLUS: { + pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64; + break; + } + case TK_MINUS: { + pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64; + break; + } + case TK_STAR: { + pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64; + break; + } + case TK_DIVIDE: { + pExpr->tokenId = TK_FLOAT; + pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; + pExpr->value.dKey = (double)pLeft->value.i64 / pRight->value.i64; + break; + } + case TK_REM: { + pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64; + break; + } + } + + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); + + } else if ((pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_INTEGER) || + (pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_FLOAT) || + (pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_FLOAT)) { + pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; + pExpr->tokenId = TK_FLOAT; + pExpr->type = SQL_NODE_VALUE; + + double left = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.dKey : pLeft->value.i64; + double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.dKey : pRight->value.i64; + + switch (optrType) { + case TK_PLUS: { + pExpr->value.dKey = left + right; + break; + } + case TK_MINUS: { + pExpr->value.dKey = left - right; + break; + } + case TK_STAR: { + pExpr->value.dKey = left * right; + break; + } + case TK_DIVIDE: { + pExpr->value.dKey = left / right; + break; + } + case TK_REM: { + pExpr->value.dKey = left - ((int64_t)(left / right)) * right; + break; + } + } + + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); + + } else { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + pExpr->pRight = pRight; + } + } else if (optrType == TK_IN) { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + + tSqlExpr *pRSub = calloc(1, sizeof(tSqlExpr)); + pRSub->tokenId = TK_SET; // TODO refactor ..... + pRSub->pParam = (SArray *)pRight; + + pExpr->pRight = pRSub; + } else { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + + if (pLeft != NULL && pRight == NULL) { + pRight = calloc(1, sizeof(tSqlExpr)); + } + + pExpr->pRight = pRight; + } + + return pExpr; +} + +tSqlExpr *tSqlExprClone(tSqlExpr *pSrc) { + tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); + + memcpy(pExpr, pSrc, sizeof(*pSrc)); + + if (pSrc->pLeft) { + pExpr->pLeft = tSqlExprClone(pSrc->pLeft); + } + + if (pSrc->pRight) { + pExpr->pRight = tSqlExprClone(pSrc->pRight); + } + + //we don't clone pParam now because clone is only used for between/and + assert(pSrc->pParam == NULL); + return pExpr; +} + +void tSqlExprCompact(tSqlExpr** pExpr) { + if (*pExpr == NULL || tSqlExprIsParentOfLeaf(*pExpr)) { + return; + } + + if ((*pExpr)->pLeft) { + tSqlExprCompact(&(*pExpr)->pLeft); + } + + if ((*pExpr)->pRight) { + tSqlExprCompact(&(*pExpr)->pRight); + } + + if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight == NULL && ((*pExpr)->tokenId == TK_OR || (*pExpr)->tokenId == TK_AND)) { + tSqlExprDestroy(*pExpr); + *pExpr = NULL; + } else if ((*pExpr)->pLeft == NULL && (*pExpr)->pRight != NULL) { + tSqlExpr* tmpPtr = (*pExpr)->pRight; + (*pExpr)->pRight = NULL; + + tSqlExprDestroy(*pExpr); + (*pExpr) = tmpPtr; + } else if ((*pExpr)->pRight == NULL && (*pExpr)->pLeft != NULL) { + tSqlExpr* tmpPtr = (*pExpr)->pLeft; + (*pExpr)->pLeft = NULL; + + tSqlExprDestroy(*pExpr); + (*pExpr) = tmpPtr; + } +} + +bool tSqlExprIsLeaf(tSqlExpr* pExpr) { + return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && + (pExpr->tokenId == 0 || + (pExpr->tokenId == TK_ID) || + (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) || + (pExpr->tokenId == TK_NULL) || + (pExpr->tokenId == TK_SET)); +} + +bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr) { + return (pExpr->pLeft != NULL && pExpr->pRight != NULL) && + (tSqlExprIsLeaf(pExpr->pLeft) && tSqlExprIsLeaf(pExpr->pRight)); +} + +static void doDestroySqlExprNode(tSqlExpr *pExpr) { + if (pExpr == NULL) { + return; + } + + if (pExpr->tokenId == TK_STRING) { + tVariantDestroy(&pExpr->value); + } + + tSqlExprListDestroy(pExpr->pParam); + free(pExpr); +} + +void tSqlExprDestroy(tSqlExpr *pExpr) { + if (pExpr == NULL) { + return; + } + + tSqlExprDestroy(pExpr->pLeft); + pExpr->pLeft = NULL; + tSqlExprDestroy(pExpr->pRight); + pExpr->pRight = NULL; + + doDestroySqlExprNode(pExpr); +} + +SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) { + if (pList == NULL) { + pList = taosArrayInit(4, sizeof(tVariantListItem)); + } + + if (pToken) { + tVariantListItem item; + tVariantCreate(&item.pVar, pToken); + item.sortOrder = order; + + taosArrayPush(pList, &item); + } + + return pList; +} + +SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) { + if (pList == NULL) { + pList = taosArrayInit(4, sizeof(tVariantListItem)); + } + + if (pVar == NULL) { + return pList; + } + + /* + * Here we do not employ the assign function, since we need the pz attribute of structure + * , which is the point to char string, to free it! + * + * Otherwise, the original pointer may be lost, which causes memory leak. + */ + tVariantListItem item; + item.pVar = *pVar; + item.sortOrder = sortOrder; + + taosArrayPush(pList, &item); + return pList; +} + +SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) { + if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) { + return tVariantListAppend(NULL, pVar, sortOrder); + } + + tVariantListItem item; + + item.pVar = *pVar; + item.sortOrder = sortOrder; + + taosArrayInsert(pList, index, &item); + return pList; +} + +SFromInfo *setTableNameList(SFromInfo* pFromInfo, SStrToken *pName, SStrToken* pAlias) { + if (pFromInfo == NULL) { + pFromInfo = calloc(1, sizeof(SFromInfo)); + pFromInfo->tableList = taosArrayInit(4, sizeof(STableNamePair)); + } + + pFromInfo->type = SQL_NODE_FROM_NAMELIST; + STableNamePair p = {.name = *pName}; + if (pAlias != NULL) { + p.aliasName = *pAlias; + } else { + TPARSER_SET_NONE_TOKEN(p.aliasName); + } + + taosArrayPush(pFromInfo->tableList, &p); + + return pFromInfo; +} + +SFromInfo *setSubquery(SFromInfo* pFromInfo, SQuerySqlNode* pSqlNode) { + if (pFromInfo == NULL) { + pFromInfo = calloc(1, sizeof(SFromInfo)); + } + + pFromInfo->type = SQL_NODE_FROM_SUBQUERY; + pFromInfo->pNode->pClause[pFromInfo->pNode->numOfClause - 1] = pSqlNode; + + return pFromInfo; +} + +void* destroyFromInfo(SFromInfo* pFromInfo) { + if (pFromInfo == NULL) { + return NULL; + } + + if (pFromInfo->type == SQL_NODE_FROM_NAMELIST) { + taosArrayDestroy(pFromInfo->tableList); + } else { + destroyAllSelectClause(pFromInfo->pNode); + } + + tfree(pFromInfo); + return NULL; +} + + +void tSetDbName(SStrToken *pCpxName, SStrToken *pDb) { + pCpxName->type = pDb->type; + pCpxName->z = pDb->z; + pCpxName->n = pDb->n; +} + +void tSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { + int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); + + // truncate the column name + if ((int32_t)pName->n >= maxLen) { + pName->n = maxLen - 1; + } + + strncpy(pField->name, pName->z, pName->n); + pField->name[pName->n] = 0; + + pField->type = pType->type; + if(!isValidDataType(pField->type)){ + pField->bytes = 0; + } else { + pField->bytes = pType->bytes; + } +} + +static int32_t tryParseNameTwoParts(SStrToken *type) { + int32_t t = -1; + + char* str = strndup(type->z, type->n); + if (str == NULL) { + return t; + } + + char* p = strtok(str, " "); + if (p == NULL) { + tfree(str); + return t; + } else { + char* unsign = strtok(NULL, " "); + if (unsign == NULL) { + tfree(str); + return t; + } + + if (strncasecmp(unsign, "UNSIGNED", 8) == 0) { + for(int32_t j = TSDB_DATA_TYPE_TINYINT; j <= TSDB_DATA_TYPE_BIGINT; ++j) { + if (strcasecmp(p, tDataTypes[j].name) == 0) { + t = j; + break; + } + } + + tfree(str); + + if (t == -1) { + return -1; + } + + switch(t) { + case TSDB_DATA_TYPE_TINYINT: return TSDB_DATA_TYPE_UTINYINT; + case TSDB_DATA_TYPE_SMALLINT: return TSDB_DATA_TYPE_USMALLINT; + case TSDB_DATA_TYPE_INT: return TSDB_DATA_TYPE_UINT; + case TSDB_DATA_TYPE_BIGINT: return TSDB_DATA_TYPE_UBIGINT; + default: + return -1; + } + + } else { + tfree(str); + return -1; + } + } +} + +void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { + // set the field type invalid + pField->type = -1; + pField->name[0] = 0; + + int32_t i = 0; + while (i < tListLen(tDataTypes)) { + if ((type->n == tDataTypes[i].nameLen) && + (strncasecmp(type->z, tDataTypes[i].name, tDataTypes[i].nameLen) == 0)) { + break; + } + + i += 1; + } + + // no qualified data type found, try unsigned data type + if (i == tListLen(tDataTypes)) { + i = tryParseNameTwoParts(type); + if (i == -1) { + return; + } + } + + pField->type = i; + pField->bytes = tDataTypes[i].bytes; + + if (i == TSDB_DATA_TYPE_NCHAR) { + /* + * for nchar, the TOKENTYPE is the number of character, so the length is the + * number of bytes in UCS-4 format, which is 4 times larger than the number of characters + */ + if (type->type == 0) { + pField->bytes = 0; + } else { + int32_t bytes = -(int32_t)(type->type); + if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + // we have to postpone reporting the error because it cannot be done here + // as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow + bytes = TSDB_MAX_NCHAR_LEN + 1; + } else { + bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } + pField->bytes = (int16_t)bytes; + } + } else if (i == TSDB_DATA_TYPE_BINARY) { + /* for binary, the TOKENTYPE is the length of binary */ + if (type->type == 0) { + pField->bytes = 0; + } else { + int32_t bytes = -(int32_t)(type->type); + if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + // refer comment for NCHAR above + bytes = TSDB_MAX_BINARY_LEN + 1; + } else { + bytes += VARSTR_HEADER_SIZE; + } + + pField->bytes = (int16_t)bytes; + } + } +} + +/* + * extract the select info out of sql string + */ +SQuerySqlNode *tSetQuerySqlNode(SStrToken *pSelectToken, SArray *pSelectList, SFromInfo *pFrom, tSqlExpr *pWhere, + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, + SSessionWindowVal *pSession, SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, + SLimitVal *psLimit) { + assert(pSelectList != NULL); + + SQuerySqlNode *pSqlNode = calloc(1, sizeof(SQuerySqlNode)); + + // all later sql string are belonged to the stream sql + pSqlNode->sqlstr = *pSelectToken; + pSqlNode->sqlstr.n = (uint32_t)strlen(pSqlNode->sqlstr.z); + + pSqlNode->pSelectList = pSelectList; + pSqlNode->from = pFrom; + pSqlNode->pGroupby = pGroupby; + pSqlNode->pSortOrder = pSortOrder; + pSqlNode->pWhere = pWhere; + pSqlNode->fillType = pFill; + + if (pLimit != NULL) { + pSqlNode->limit = *pLimit; + } else { + pSqlNode->limit.limit = -1; + pSqlNode->limit.offset = 0; + } + + if (psLimit != NULL) { + pSqlNode->slimit = *psLimit; + } else { + pSqlNode->slimit.limit = -1; + pSqlNode->slimit.offset = 0; + } + + if (pInterval != NULL) { + pSqlNode->interval = *pInterval; + } else { + TPARSER_SET_NONE_TOKEN(pSqlNode->interval.interval); + TPARSER_SET_NONE_TOKEN(pSqlNode->interval.offset); + } + + if (pSliding != NULL) { + pSqlNode->sliding = *pSliding; + } else { + TPARSER_SET_NONE_TOKEN(pSqlNode->sliding); + } + + if (pSession != NULL) { + pSqlNode->sessionVal = *pSession; + } else { + TPARSER_SET_NONE_TOKEN(pSqlNode->sessionVal.gap); + TPARSER_SET_NONE_TOKEN(pSqlNode->sessionVal.col); + } + + return pSqlNode; +} + +static void freeVariant(void *pItem) { + tVariantListItem* p = (tVariantListItem*) pItem; + tVariantDestroy(&p->pVar); +} + +void freeCreateTableInfo(void* p) { + SCreatedTableInfo* pInfo = (SCreatedTableInfo*) p; + taosArrayDestroy(pInfo->pTagNames); + taosArrayDestroyEx(pInfo->pTagVals, freeVariant); + tfree(pInfo->fullname); + tfree(pInfo->tagdata.data); +} + +void destroyQuerySqlNode(SQuerySqlNode *pQuerySql) { + if (pQuerySql == NULL) { + return; + } + + tSqlExprListDestroy(pQuerySql->pSelectList); + + pQuerySql->pSelectList = NULL; + + tSqlExprDestroy(pQuerySql->pWhere); + pQuerySql->pWhere = NULL; + + taosArrayDestroyEx(pQuerySql->pSortOrder, freeVariant); + pQuerySql->pSortOrder = NULL; + + taosArrayDestroyEx(pQuerySql->pGroupby, freeVariant); + pQuerySql->pGroupby = NULL; + + pQuerySql->from = destroyFromInfo(pQuerySql->from); + + taosArrayDestroyEx(pQuerySql->fillType, freeVariant); + pQuerySql->fillType = NULL; + + free(pQuerySql); +} + +void destroyAllSelectClause(SSubclauseInfo *pClause) { + if (pClause == NULL || pClause->numOfClause == 0) { + return; + } + + for(int32_t i = 0; i < pClause->numOfClause; ++i) { + SQuerySqlNode *pQuerySql = pClause->pClause[i]; + destroyQuerySqlNode(pQuerySql); + } + + tfree(pClause->pClause); +} + +SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SQuerySqlNode *pSelect, int32_t type) { + SCreateTableSql *pCreate = calloc(1, sizeof(SCreateTableSql)); + + switch (type) { + case TSQL_CREATE_TABLE: { + pCreate->colInfo.pColumns = pCols; + assert(pTags == NULL); + break; + } + case TSQL_CREATE_STABLE: { + pCreate->colInfo.pColumns = pCols; + pCreate->colInfo.pTagColumns = pTags; + assert(pTags != NULL && pCols != NULL); + break; + } + case TSQL_CREATE_STREAM: { + pCreate->pSelect = pSelect; + break; + } + + case TSQL_CREATE_TABLE_FROM_STABLE: { + assert(0); + } + + default: + assert(false); + } + + pCreate->type = type; + return pCreate; +} + +SCreatedTableInfo createNewChildTableInfo(SStrToken *pTableName, SArray *pTagNames, SArray *pTagVals, SStrToken *pToken, SStrToken* igExists) { + SCreatedTableInfo info; + memset(&info, 0, sizeof(SCreatedTableInfo)); + + info.name = *pToken; + info.pTagNames = pTagNames; + info.pTagVals = pTagVals; + info.stableName = *pTableName; + info.igExist = (igExists->n > 0)? 1:0; + + return info; +} + +SAlterTableInfo *tSetAlterTableInfo(SStrToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, int16_t tableType) { + SAlterTableInfo *pAlterTable = calloc(1, sizeof(SAlterTableInfo)); + + pAlterTable->name = *pTableName; + pAlterTable->type = type; + pAlterTable->tableType = tableType; + + if (type == TSDB_ALTER_TABLE_ADD_COLUMN || type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) { + pAlterTable->pAddColumns = pCols; + assert(pVals == NULL); + } else { + /* + * ALTER_TABLE_TAGS_CHG, ALTER_TABLE_TAGS_SET, ALTER_TABLE_TAGS_DROP, + * ALTER_TABLE_DROP_COLUMN + */ + pAlterTable->varList = pVals; + assert(pCols == NULL); + } + + return pAlterTable; +} + +void* destroyCreateTableSql(SCreateTableSql* pCreate) { + destroyQuerySqlNode(pCreate->pSelect); + + taosArrayDestroy(pCreate->colInfo.pColumns); + taosArrayDestroy(pCreate->colInfo.pTagColumns); + + taosArrayDestroyEx(pCreate->childTableInfo, freeCreateTableInfo); + tfree(pCreate); + + return NULL; +} + +void SqlInfoDestroy(SSqlInfo *pInfo) { + if (pInfo == NULL) return; + + if (pInfo->type == TSDB_SQL_SELECT) { + destroyAllSelectClause(&pInfo->subclauseInfo); + } else if (pInfo->type == TSDB_SQL_CREATE_TABLE) { + pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo); + } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { + taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant); + taosArrayDestroy(pInfo->pAlterInfo->pAddColumns); + tfree(pInfo->pAlterInfo->tagData.data); + tfree(pInfo->pAlterInfo); + } else { + if (pInfo->pMiscInfo != NULL) { + taosArrayDestroy(pInfo->pMiscInfo->a); + } + + if (pInfo->pMiscInfo != NULL && pInfo->type == TSDB_SQL_CREATE_DB) { + taosArrayDestroyEx(pInfo->pMiscInfo->dbOpt.keep, freeVariant); + } + + tfree(pInfo->pMiscInfo); + } +} + +SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { + if (pSubclause == NULL) { + pSubclause = calloc(1, sizeof(SSubclauseInfo)); + } + + int32_t newSize = pSubclause->numOfClause + 1; + char* tmp = realloc(pSubclause->pClause, newSize * POINTER_BYTES); + if (tmp == NULL) { + return pSubclause; + } + + pSubclause->pClause = (SQuerySqlNode**) tmp; + + pSubclause->pClause[newSize - 1] = pSqlExprInfo; + pSubclause->numOfClause++; + + return pSubclause; +} + +SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pTableName, int32_t type) { + pInfo->type = type; + + if (type == TSDB_SQL_SELECT) { + pInfo->subclauseInfo = *(SSubclauseInfo*) pSqlExprInfo; + free(pSqlExprInfo); + } else { + pInfo->pCreateTableInfo = pSqlExprInfo; + } + + if (pTableName != NULL) { + pInfo->pCreateTableInfo->name = *pTableName; + } + + return pInfo; +} + +SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) { + char* tmp = realloc(pQueryInfo->pClause, (pQueryInfo->numOfClause + 1) * POINTER_BYTES); + if (tmp == NULL) { // out of memory + return pQueryInfo; + } + + pQueryInfo->pClause = (SQuerySqlNode**) tmp; + pQueryInfo->pClause[pQueryInfo->numOfClause++] = pSubclause; + + return pQueryInfo; +} + +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pTableNameToken, SStrToken *pIfNotExists) { + pInfo->pCreateTableInfo->name = *pTableNameToken; + pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); +} + +void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { + pInfo->type = type; + if (nParam == 0) { + return; + } + + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); + pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); + } + + va_list va; + va_start(va, nParam); + + while ((nParam--) > 0) { + SStrToken *pToken = va_arg(va, SStrToken *); + taosArrayPush(pInfo->pMiscInfo->a, pToken); + } + + va_end(va); +} + +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck, int16_t dbType, int16_t tableType) { + pInfo->type = type; + + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo)); + pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken)); + } + + taosArrayPush(pInfo->pMiscInfo->a, pToken); + + pInfo->pMiscInfo->existsCheck = (existsCheck->n == 1); + pInfo->pMiscInfo->dbType = dbType; + pInfo->pMiscInfo->tableType = tableType; +} + +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); + } + + pInfo->type = TSDB_SQL_SHOW; + + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; + pShowInfo->showType = type; + + if (prefix != NULL && prefix->type != 0) { + pShowInfo->prefix = *prefix; + } else { + pShowInfo->prefix.type = 0; + } + + if (pPatterns != NULL && pPatterns->type != 0) { + pShowInfo->pattern = *pPatterns; + } else { + pShowInfo->pattern.type = 0; + } +} + +void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDbInfo *pDB, SStrToken *pIgExists) { + pInfo->type = type; + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); + } + + pInfo->pMiscInfo->dbOpt = *pDB; + pInfo->pMiscInfo->dbOpt.dbname = *pToken; + pInfo->pMiscInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} +} + +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo) { + pInfo->type = type; + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); + } + + pInfo->pMiscInfo->acctOpt = *pAcctInfo; + + assert(pName != NULL); + pInfo->pMiscInfo->user.user = *pName; + + if (pPwd != NULL) { + pInfo->pMiscInfo->user.passwd = *pPwd; + } +} + +void setCreateUserSql(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { + pInfo->type = TSDB_SQL_CREATE_USER; + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); + } + + assert(pName != NULL && pPasswd != NULL); + + pInfo->pMiscInfo->user.user = *pName; + pInfo->pMiscInfo->user.passwd = *pPasswd; +} + +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { + pInfo->type = TSDB_SQL_ALTER_USER; + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); + } + + assert(pName != NULL); + + SUserInfo* pUser = &pInfo->pMiscInfo->user; + pUser->type = type; + pUser->user = *pName; + + if (pPwd != NULL) { + pUser->passwd = *pPwd; + } else { + pUser->passwd.type = TSDB_DATA_TYPE_NULL; + } + + if (pPrivilege != NULL) { + pUser->privilege = *pPrivilege; + } else { + pUser->privilege.type = TSDB_DATA_TYPE_NULL; + } +} + +void setKillSql(SSqlInfo *pInfo, int32_t type, SStrToken *id) { + pInfo->type = type; + if (pInfo->pMiscInfo == NULL) { + pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo)); + } + + assert(id != NULL); + pInfo->pMiscInfo->id = *id; +} + +void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) { + pDBInfo->compressionLevel = -1; + + pDBInfo->walLevel = -1; + pDBInfo->fsyncPeriod = -1; + pDBInfo->commitTime = -1; + pDBInfo->maxTablesPerVnode = -1; + + pDBInfo->cacheBlockSize = -1; + pDBInfo->numOfBlocks = -1; + pDBInfo->maxRowsPerBlock = -1; + pDBInfo->minRowsPerBlock = -1; + pDBInfo->daysPerFile = -1; + + pDBInfo->replica = -1; + pDBInfo->quorum = -1; + pDBInfo->keep = NULL; + + pDBInfo->update = -1; + pDBInfo->cachelast = -1; + + pDBInfo->dbType = -1; + pDBInfo->partitions = -1; + + memset(&pDBInfo->precision, 0, sizeof(SStrToken)); +} + +void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo) { + setDefaultCreateDbOption(pDBInfo); + + pDBInfo->dbType = TSDB_DB_TYPE_TOPIC; + pDBInfo->partitions = TSDB_DEFAULT_DB_PARTITON_OPTION; +} diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 48d2f5d505557355b851f6a107d6b225729ee64f..52b2fdbb8279f3329862689675f12aee9b64cfd5 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -100,6 +100,7 @@ static SKeyword keywordTable[] = { {"ACCOUNT", TK_ACCOUNT}, {"USE", TK_USE}, {"DESCRIBE", TK_DESCRIBE}, + {"SYNCDB", TK_SYNCDB}, {"ALTER", TK_ALTER}, {"PASS", TK_PASS}, {"PRIVILEGE", TK_PRIVILEGE}, @@ -139,6 +140,7 @@ static SKeyword keywordTable[] = { {"FROM", TK_FROM}, {"VARIABLE", TK_VARIABLE}, {"INTERVAL", TK_INTERVAL}, + {"SESSION", TK_SESSION}, {"FILL", TK_FILL}, {"SLIDING", TK_SLIDING}, {"ORDER", TK_ORDER}, @@ -200,46 +202,22 @@ static SKeyword keywordTable[] = { {"TRIGGER", TK_TRIGGER}, {"VIEW", TK_VIEW}, {"ALL", TK_ALL}, - {"COUNT", TK_COUNT}, - {"SUM", TK_SUM}, - {"AVG", TK_AVG}, - {"MIN", TK_MIN}, - {"MAX", TK_MAX}, - {"FIRST", TK_FIRST}, - {"LAST", TK_LAST}, - {"TOP", TK_TOP}, - {"BOTTOM", TK_BOTTOM}, - {"STDDEV", TK_STDDEV}, - {"PERCENTILE", TK_PERCENTILE}, - {"APERCENTILE", TK_APERCENTILE}, - {"LEASTSQUARES", TK_LEASTSQUARES}, - {"HISTOGRAM", TK_HISTOGRAM}, - {"DIFF", TK_DIFF}, - {"SPREAD", TK_SPREAD}, - {"TWA", TK_TWA}, - {"INTERP", TK_INTERP}, - {"LAST_ROW", TK_LAST_ROW}, {"SEMI", TK_SEMI}, {"NONE", TK_NONE}, {"PREV", TK_PREV}, {"LINEAR", TK_LINEAR}, {"IMPORT", TK_IMPORT}, - {"METRIC", TK_METRIC}, {"TBNAME", TK_TBNAME}, {"JOIN", TK_JOIN}, - {"METRICS", TK_METRICS}, - {"TBID", TK_TBID}, {"STABLE", TK_STABLE}, {"FILE", TK_FILE}, {"VNODES", TK_VNODES}, {"UNION", TK_UNION}, - {"RATE", TK_RATE}, - {"IRATE", TK_IRATE}, - {"SUM_RATE", TK_SUM_RATE}, - {"SUM_IRATE", TK_SUM_IRATE}, - {"AVG_RATE", TK_AVG_RATE}, - {"AVG_IRATE", TK_AVG_IRATE}, {"CACHELAST", TK_CACHELAST}, + {"DISTINCT", TK_DISTINCT}, + {"PARTITIONS", TK_PARTITIONS}, + {"TOPIC", TK_TOPIC}, + {"TOPICS", TK_TOPICS} }; static const char isIdChar[] = { @@ -293,7 +271,7 @@ int tSQLKeywordCode(const char* z, int n) { * Return the length of the token that begins at z[0]. * Store the token type in *type before returning. */ -uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { +uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { uint32_t i; switch (*z) { case ' ': @@ -303,121 +281,121 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { case '\r': { for (i = 1; isspace(z[i]); i++) { } - *tokenType = TK_SPACE; + *tokenId = TK_SPACE; return i; } case ':': { - *tokenType = TK_COLON; + *tokenId = TK_COLON; return 1; } case '-': { if (z[1] == '-') { for (i = 2; z[i] && z[i] != '\n'; i++) { } - *tokenType = TK_COMMENT; + *tokenId = TK_COMMENT; return i; } - *tokenType = TK_MINUS; + *tokenId = TK_MINUS; return 1; } case '(': { - *tokenType = TK_LP; + *tokenId = TK_LP; return 1; } case ')': { - *tokenType = TK_RP; + *tokenId = TK_RP; return 1; } case ';': { - *tokenType = TK_SEMI; + *tokenId = TK_SEMI; return 1; } case '+': { - *tokenType = TK_PLUS; + *tokenId = TK_PLUS; return 1; } case '*': { - *tokenType = TK_STAR; + *tokenId = TK_STAR; return 1; } case '/': { if (z[1] != '*' || z[2] == 0) { - *tokenType = TK_SLASH; + *tokenId = TK_SLASH; return 1; } for (i = 3; z[i] && (z[i] != '/' || z[i - 1] != '*'); i++) { } if (z[i]) i++; - *tokenType = TK_COMMENT; + *tokenId = TK_COMMENT; return i; } case '%': { - *tokenType = TK_REM; + *tokenId = TK_REM; return 1; } case '=': { - *tokenType = TK_EQ; + *tokenId = TK_EQ; return 1 + (z[1] == '='); } case '<': { if (z[1] == '=') { - *tokenType = TK_LE; + *tokenId = TK_LE; return 2; } else if (z[1] == '>') { - *tokenType = TK_NE; + *tokenId = TK_NE; return 2; } else if (z[1] == '<') { - *tokenType = TK_LSHIFT; + *tokenId = TK_LSHIFT; return 2; } else { - *tokenType = TK_LT; + *tokenId = TK_LT; return 1; } } case '>': { if (z[1] == '=') { - *tokenType = TK_GE; + *tokenId = TK_GE; return 2; } else if (z[1] == '>') { - *tokenType = TK_RSHIFT; + *tokenId = TK_RSHIFT; return 2; } else { - *tokenType = TK_GT; + *tokenId = TK_GT; return 1; } } case '!': { if (z[1] != '=') { - *tokenType = TK_ILLEGAL; + *tokenId = TK_ILLEGAL; return 2; } else { - *tokenType = TK_NE; + *tokenId = TK_NE; return 2; } } case '|': { if (z[1] != '|') { - *tokenType = TK_BITOR; + *tokenId = TK_BITOR; return 1; } else { - *tokenType = TK_CONCAT; + *tokenId = TK_CONCAT; return 2; } } case ',': { - *tokenType = TK_COMMA; + *tokenId = TK_COMMA; return 1; } case '&': { - *tokenType = TK_BITAND; + *tokenId = TK_BITAND; return 1; } case '~': { - *tokenType = TK_BITNOT; + *tokenId = TK_BITNOT; return 1; } case '?': { - *tokenType = TK_QUESTION; + *tokenId = TK_QUESTION; return 1; } case '\'': @@ -443,7 +421,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { if (z[i]) i++; if (strEnd) { - *tokenType = TK_STRING; + *tokenId = TK_STRING; return i; } @@ -467,10 +445,10 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { } } - *tokenType = TK_FLOAT; + *tokenId = TK_FLOAT; return i; } else { - *tokenType = TK_DOT; + *tokenId = TK_DOT; return 1; } } @@ -479,7 +457,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { char next = z[1]; if (next == 'b') { // bin number - *tokenType = TK_BIN; + *tokenId = TK_BIN; for (i = 2; (z[i] == '0' || z[i] == '1'); ++i) { } @@ -489,7 +467,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { return i; } else if (next == 'x') { //hex number - *tokenType = TK_HEX; + *tokenId = TK_HEX; for (i = 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) { } @@ -509,7 +487,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { case '7': case '8': case '9': { - *tokenType = TK_INTEGER; + *tokenId = TK_INTEGER; for (i = 1; isdigit(z[i]); i++) { } @@ -519,7 +497,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { z[i] == 'U' || z[i] == 'A' || z[i] == 'S' || z[i] == 'M' || z[i] == 'H' || z[i] == 'D' || z[i] == 'N' || z[i] == 'Y' || z[i] == 'W') && (isIdChar[(uint8_t)z[i + 1]] == 0)) { - *tokenType = TK_VARIABLE; + *tokenId = TK_VARIABLE; i += 1; return i; } @@ -530,12 +508,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { while (isdigit(z[i])) { i++; } - *tokenType = TK_FLOAT; + *tokenId = TK_FLOAT; seg++; } if (seg == 4) { // ip address - *tokenType = TK_IPTOKEN; + *tokenId = TK_IPTOKEN; return i; } @@ -545,14 +523,14 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { while (isdigit(z[i])) { i++; } - *tokenType = TK_FLOAT; + *tokenId = TK_FLOAT; } return i; } case '[': { for (i = 1; z[i] && z[i - 1] != ']'; i++) { } - *tokenType = TK_ID; + *tokenId = TK_ID; return i; } case 'T': @@ -563,7 +541,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { } if ((i == 4 && strncasecmp(z, "true", 4) == 0) || (i == 5 && strncasecmp(z, "false", 5) == 0)) { - *tokenType = TK_BOOL; + *tokenId = TK_BOOL; return i; } } @@ -573,12 +551,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { } for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t) z[i]]; i++) { } - *tokenType = tSQLKeywordCode(z, i); + *tokenId = tSQLKeywordCode(z, i); return i; } } - *tokenType = TK_ILLEGAL; + *tokenId = TK_ILLEGAL; return 0; } diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index a53f8935ee9b941a10acab4d59a08db422d4b692..1f43c5b33cca044744a2783a251f76bcb827d34a 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -23,7 +23,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { pTSBuf->autoDelete = autoDelete; taosGetTmpfilePath("join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "w+"); + pTSBuf->f = fopen(pTSBuf->path, "wb+"); if (pTSBuf->f == NULL) { free(pTSBuf); return NULL; @@ -59,7 +59,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - pTSBuf->f = fopen(pTSBuf->path, "r+"); + pTSBuf->f = fopen(pTSBuf->path, "rb+"); if (pTSBuf->f == NULL) { free(pTSBuf); return NULL; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index dc01de0f92a0d591c67f3a156caaae34e318c962..da7dbcd501373f34f67e25595af2ea88c8d07f4a 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -19,6 +19,16 @@ #include "qExecutor.h" #include "qUtil.h" +#include "tbuffer.h" +#include "tlosertree.h" +#include "queryLog.h" +#include "tscompression.h" + +typedef struct SCompSupporter { + STableQueryInfo **pTableQueryInfo; + int32_t *rowIndex; + int32_t order; +} SCompSupporter; int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; @@ -32,12 +42,11 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { } int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { - pResultRowInfo->capacity = size; - - pResultRowInfo->type = type; - pResultRowInfo->curIndex = -1; + pResultRowInfo->type = type; pResultRowInfo->size = 0; pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; + pResultRowInfo->curIndex = -1; + pResultRowInfo->capacity = size; pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); if (pResultRowInfo->pResult == NULL) { @@ -126,20 +135,22 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 if (pResultRow->pageId >= 0) { tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); + int16_t offset = 0; for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[i]; - char * s = getPosInResultPage(pRuntimeEnv, i, pResultRow, page); - size_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes; + int16_t size = pRuntimeEnv->pQuery->pExpr1[i].bytes; + char * s = getPosInResultPage(pRuntimeEnv->pQuery, page, pResultRow->offset, offset); memset(s, 0, size); + offset += size; RESET_RESULT_INFO(pResultInfo); } } pResultRow->numOfRows = 0; pResultRow->pageId = -1; - pResultRow->rowId = -1; + pResultRow->offset = -1; pResultRow->closed = false; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { @@ -149,13 +160,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 } } -SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) { - assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput); - return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]); +// TODO refactor: use macro +SResultRowCellInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset) { + assert(index >= 0 && offset != NULL); + return (SResultRowCellInfo*)((char*) pRow->pCellInfo + offset[index]); } size_t getResultRowSize(SQueryRuntimeEnv* pRuntimeEnv) { - return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow); + SQuery* pQuery = pRuntimeEnv->pQuery; + return (pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pQuery->interBufSize + sizeof(SResultRow); } SResultRowPool* initResultRowPool(size_t size) { @@ -228,4 +241,409 @@ void* destroyResultRowPool(SResultRowPool* p) { tfree(p); return NULL; -} \ No newline at end of file +} + +void interResToBinary(SBufferWriter* bw, SArray* pRes, int32_t tagLen) { + uint32_t numOfGroup = (uint32_t) taosArrayGetSize(pRes); + tbufWriteUint32(bw, numOfGroup); + tbufWriteUint16(bw, tagLen); + + for(int32_t i = 0; i < numOfGroup; ++i) { + SInterResult* pOne = taosArrayGet(pRes, i); + if (tagLen > 0) { + tbufWriteBinary(bw, pOne->tags, tagLen); + } + + uint32_t numOfCols = (uint32_t) taosArrayGetSize(pOne->pResult); + tbufWriteUint32(bw, numOfCols); + for(int32_t j = 0; j < numOfCols; ++j) { + SStddevInterResult* p = taosArrayGet(pOne->pResult, j); + uint32_t numOfRows = (uint32_t) taosArrayGetSize(p->pResult); + + tbufWriteUint16(bw, p->colId); + tbufWriteUint32(bw, numOfRows); + + for(int32_t k = 0; k < numOfRows; ++k) { + SResPair v = *(SResPair*) taosArrayGet(p->pResult, k); + tbufWriteDouble(bw, v.avg); + tbufWriteInt64(bw, v.key); + } + } + } +} + +SArray* interResFromBinary(const char* data, int32_t len) { + SBufferReader br = tbufInitReader(data, len, false); + uint32_t numOfGroup = tbufReadUint32(&br); + uint16_t tagLen = tbufReadUint16(&br); + + char* tag = NULL; + if (tagLen > 0) { + tag = calloc(1, tagLen); + } + + SArray* pResult = taosArrayInit(4, sizeof(SInterResult)); + + for(int32_t i = 0; i < numOfGroup; ++i) { + if (tagLen > 0) { + memset(tag, 0, tagLen); + tbufReadToBinary(&br, tag, tagLen); + } + + uint32_t numOfCols = tbufReadUint32(&br); + + SArray* p = taosArrayInit(numOfCols, sizeof(SStddevInterResult)); + for(int32_t j = 0; j < numOfCols; ++j) { + int16_t colId = tbufReadUint16(&br); + int32_t numOfRows = tbufReadUint32(&br); + + SStddevInterResult interRes = {.colId = colId, .pResult = taosArrayInit(4, sizeof(struct SResPair)),}; + for(int32_t k = 0; k < numOfRows; ++k) { + SResPair px = {0}; + px.avg = tbufReadDouble(&br); + px.key = tbufReadInt64(&br); + + taosArrayPush(interRes.pResult, &px); + } + + taosArrayPush(p, &interRes); + } + + char* p1 = NULL; + if (tagLen > 0) { + p1 = malloc(tagLen); + memcpy(p1, tag, tagLen); + } + + SInterResult d = {.pResult = p, .tags = p1,}; + taosArrayPush(pResult, &d); + } + + tfree(tag); + return pResult; +} + +void freeInterResult(void* param) { + SInterResult* pResult = (SInterResult*) param; + tfree(pResult->tags); + + int32_t numOfCols = (int32_t) taosArrayGetSize(pResult->pResult); + for(int32_t i = 0; i < numOfCols; ++i) { + SStddevInterResult *p = taosArrayGet(pResult->pResult, i); + taosArrayDestroy(p->pResult); + } + + taosArrayDestroy(pResult->pResult); +} + +void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { + assert(pGroupResInfo != NULL); + + taosArrayDestroy(pGroupResInfo->pRows); + pGroupResInfo->pRows = NULL; + pGroupResInfo->index = 0; +} + +void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) { + if (pGroupResInfo->pRows != NULL) { + taosArrayDestroy(pGroupResInfo->pRows); + } + + pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pResult, pResultInfo->size, POINTER_BYTES); + pGroupResInfo->index = 0; + + assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); +} + +bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo) { + if (pGroupResInfo->pRows == NULL) { + return false; + } + + return pGroupResInfo->index < taosArrayGetSize(pGroupResInfo->pRows); +} + +bool hasRemainData(SGroupResInfo* pGroupResInfo) { + if (hasRemainDataInCurrentGroup(pGroupResInfo)) { + return true; + } + + return pGroupResInfo->currentGroup < pGroupResInfo->totalGroup; +} + +bool incNextGroup(SGroupResInfo* pGroupResInfo) { + return (++pGroupResInfo->currentGroup) < pGroupResInfo->totalGroup; +} + +int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { + assert(pGroupResInfo != NULL); + if (pGroupResInfo->pRows == 0) { + return 0; + } + + return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); +} + +static int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow, int32_t* rowCellInfoOffset) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { + int32_t functionId = pQuery->pExpr1[j].base.functionId; + + /* + * ts, tag, tagprj function can not decide the output number of current query + * the number of output result is decided by main output + */ + if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) { + continue; + } + + SResultRowCellInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset); + assert(pResultInfo != NULL); + + if (pResultInfo->numOfRes > 0) { + return pResultInfo->numOfRes; + } + } + + return 0; +} + +static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { + int32_t left = *(int32_t *)pLeft; + int32_t right = *(int32_t *)pRight; + + SCompSupporter * supporter = (SCompSupporter *)param; + + int32_t leftPos = supporter->rowIndex[left]; + int32_t rightPos = supporter->rowIndex[right]; + + /* left source is exhausted */ + if (leftPos == -1) { + return 1; + } + + /* right source is exhausted*/ + if (rightPos == -1) { + return -1; + } + + STableQueryInfo** pList = supporter->pTableQueryInfo; + + SResultRowInfo *pWindowResInfo1 = &(pList[left]->resInfo); + SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); + TSKEY leftTimestamp = pWindowRes1->win.skey; + + SResultRowInfo *pWindowResInfo2 = &(pList[right]->resInfo); + SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); + TSKEY rightTimestamp = pWindowRes2->win.skey; + + if (leftTimestamp == rightTimestamp) { + return 0; + } + + if (supporter->order == TSDB_ORDER_ASC) { + return (leftTimestamp > rightTimestamp)? 1:-1; + } else { + return (leftTimestamp < rightTimestamp)? 1:-1; + } +} + +static int32_t mergeIntoGroupResultImpl(SQueryRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList, + int32_t* rowCellInfoOffset) { + bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery); + + int32_t code = TSDB_CODE_SUCCESS; + + int32_t *posList = NULL; + SLoserTreeInfo *pTree = NULL; + STableQueryInfo **pTableQueryInfoList = NULL; + + size_t size = taosArrayGetSize(pTableList); + if (pGroupResInfo->pRows == NULL) { + pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES); + } + + posList = calloc(size, sizeof(int32_t)); + pTableQueryInfoList = malloc(POINTER_BYTES * size); + + if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { + qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; + } + + int32_t numOfTables = 0; + for (int32_t i = 0; i < size; ++i) { + STableQueryInfo *item = taosArrayGetP(pTableList, i); + if (item->resInfo.size > 0) { + pTableQueryInfoList[numOfTables++] = item; + } + } + + // there is no data in current group + // no need to merge results since only one table in each group + if (numOfTables == 0) { + goto _end; + } + + SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQuery->order.order}; + + int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); + if (ret != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _end; + } + + int64_t lastTimestamp = ascQuery? INT64_MIN:INT64_MAX; + int64_t startt = taosGetTimestampMs(); + + while (1) { + int32_t tableIndex = pTree->pNode[0].index; + + SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo; + SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); + + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes, rowCellInfoOffset); + if (num <= 0) { + cs.rowIndex[tableIndex] += 1; + + if (cs.rowIndex[tableIndex] >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; + if (--numOfTables == 0) { // all input sources are exhausted + break; + } + } + } else { + assert((pWindowRes->win.skey >= lastTimestamp && ascQuery) || (pWindowRes->win.skey <= lastTimestamp && !ascQuery)); + + if (pWindowRes->win.skey != lastTimestamp) { + taosArrayPush(pGroupResInfo->pRows, &pWindowRes); + pWindowRes->numOfRows = (uint32_t) num; + } + + lastTimestamp = pWindowRes->win.skey; + + // move to the next row of current entry + if ((++cs.rowIndex[tableIndex]) >= pWindowResInfo->size) { + cs.rowIndex[tableIndex] = -1; + + // all input sources are exhausted + if ((--numOfTables) == 0) { + break; + } + } + } + + tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries); + } + + int64_t endt = taosGetTimestampMs(); + + qDebug("QInfo:%"PRIu64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), + pGroupResInfo->currentGroup, endt - startt); + + _end: + tfree(pTableQueryInfoList); + tfree(posList); + tfree(pTree); + + return code; +} + +int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t* offset) { + int64_t st = taosGetTimestampUs(); + + while (pGroupResInfo->currentGroup < pGroupResInfo->totalGroup) { + SArray *group = GET_TABLEGROUP(pRuntimeEnv, pGroupResInfo->currentGroup); + + int32_t ret = mergeIntoGroupResultImpl(pRuntimeEnv, pGroupResInfo, group, offset); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + // this group generates at least one result, return results + if (taosArrayGetSize(pGroupResInfo->pRows) > 0) { + break; + } + + qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); + cleanupGroupResInfo(pGroupResInfo); + incNextGroup(pGroupResInfo); + } + + int64_t elapsedTime = taosGetTimestampUs() - st; + qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), + pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); + +// pQInfo->summary.firstStageMergeTime += elapsedTime; + return TSDB_CODE_SUCCESS; +} + +void blockDistInfoToBinary(STableBlockDist* pDist, struct SBufferWriter* bw) { + tbufWriteUint32(bw, pDist->numOfTables); + tbufWriteUint16(bw, pDist->numOfFiles); + tbufWriteUint64(bw, pDist->totalSize); + tbufWriteUint32(bw, pDist->numOfRowsInMemTable); + tbufWriteUint64(bw, taosArrayGetSize(pDist->dataBlockInfos)); + + // compress the binary string + char* p = TARRAY_GET_START(pDist->dataBlockInfos); + + // compress extra bytes + size_t x = taosArrayGetSize(pDist->dataBlockInfos) * pDist->dataBlockInfos->elemSize; + char* tmp = malloc(x + 2); + + bool comp = false; + int32_t len = tsCompressString(p, (int32_t)x, 1, tmp, (int32_t)x, ONE_STAGE_COMP, NULL, 0); + if (len == -1 || len >= x) { // compress failed, do not compress this binary data + comp = false; + len = (int32_t)x; + } else { + comp = true; + } + + tbufWriteUint8(bw, comp); + tbufWriteUint32(bw, len); + if (comp) { + tbufWriteBinary(bw, tmp, len); + } else { + tbufWriteBinary(bw, p, len); + } + tfree(tmp); +} + +void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) { + SBufferReader br = tbufInitReader(data, len, false); + + pDist->numOfTables = tbufReadUint32(&br); + pDist->numOfFiles = tbufReadUint16(&br); + pDist->totalSize = tbufReadUint64(&br); + pDist->numOfRowsInMemTable = tbufReadUint32(&br); + int64_t numOfBlocks = tbufReadUint64(&br); + + bool comp = tbufReadUint8(&br); + uint32_t compLen = tbufReadUint32(&br); + + size_t originalLen = (size_t) (numOfBlocks*sizeof(SFileBlockInfo)); + + char* outputBuf = NULL; + if (comp) { + outputBuf = malloc(originalLen); + + size_t actualLen = compLen; + const char* compStr = tbufReadBinary(&br, &actualLen); + + int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf, + (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0); + assert(orignalLen == numOfBlocks*sizeof(SFileBlockInfo)); + } else { + outputBuf = (char*) tbufReadBinary(&br, &originalLen); + } + + pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t) numOfBlocks, sizeof(SFileBlockInfo)); + if (comp) { + tfree(outputBuf); + } +} + diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c new file mode 100644 index 0000000000000000000000000000000000000000..2ff0c9676e72074874c977899599c6666715c2ec --- /dev/null +++ b/src/query/src/queryMain.c @@ -0,0 +1,537 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "qFill.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tglobal.h" + +#include "exception.h" +#include "hash.h" +#include "texpr.h" +#include "qExecutor.h" +#include "qResultbuf.h" +#include "qUtil.h" +#include "query.h" +#include "queryLog.h" +#include "tlosertree.h" +#include "ttype.h" +#include "tcompare.h" + +typedef struct SQueryMgmt { + pthread_mutex_t lock; + SCacheObj *qinfoPool; // query handle pool + int32_t vgId; + bool closed; +} SQueryMgmt; + +static void queryMgmtKillQueryFn(void* handle) { + void** fp = (void**)handle; + qKillQuery(*fp); +} + +static void freeqinfoFn(void *qhandle) { + void** handle = qhandle; + if (handle == NULL || *handle == NULL) { + return; + } + + qKillQuery(*handle); + qDestroyQueryInfo(*handle); +} + +void freeParam(SQueryParam *param) { + tfree(param->sql); + tfree(param->tagCond); + tfree(param->tbnameCond); + tfree(param->pTableIdList); + tfree(param->pExprMsg); + tfree(param->pSecExprMsg); + tfree(param->pExprs); + tfree(param->pSecExprs); + tfree(param->pGroupColIndex); + tfree(param->pTagColumnInfo); + tfree(param->pGroupbyExpr); + tfree(param->prevResult); +} + +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t *qId) { + assert(pQueryMsg != NULL && tsdb != NULL); + + int32_t code = TSDB_CODE_SUCCESS; + + SQueryParam param = {0}; + code = convertQueryMsg(pQueryMsg, ¶m); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } + + if (pQueryMsg->numOfTables <= 0) { + qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _over; + } + + if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { + qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); + code = TSDB_CODE_QRY_INVALID_MSG; + goto _over; + } + + if ((code = createQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExprMsg, param.pTagColumnInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + + if (param.pSecExprMsg != NULL) { + if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExprMsg, param.pExprs)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } + + param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); + if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { + goto _over; + } + + bool isSTableQuery = false; + STableGroupInfo tableGroupInfo = {0}; + int64_t st = taosGetTimestampUs(); + + if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { + STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); + + qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); + if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { + goto _over; + } + } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { + isSTableQuery = true; + + // also note there's possibility that only one table in the super table + if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { + STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); + + // group by normal column, do not pass the group by condition to tsdb to group table into different group + int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; + if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { + numOfGroupByCols = 0; + } + + qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); + code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, + pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); + + if (code != TSDB_CODE_SUCCESS) { + qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); + goto _over; + } + } else { + code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _over; + } + + qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); + } + + int64_t el = taosGetTimestampUs() - st; + qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); + } else { + assert(0); + } + + code = checkForQueryBuf(tableGroupInfo.numOfTables); + if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort + goto _over; + } + + (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, param.pTagColumnInfo, isSTableQuery, param.sql, qId); + + param.sql = NULL; + param.pExprs = NULL; + param.pSecExprs = NULL; + param.pGroupbyExpr = NULL; + param.pTagColumnInfo = NULL; + + if ((*pQInfo) == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _over; + } + + code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, ¶m, isSTableQuery); + + _over: + if (param.pGroupbyExpr != NULL) { + taosArrayDestroy(param.pGroupbyExpr->columnInfo); + } + + taosArrayDestroy(param.pTableIdList); + param.pTableIdList = NULL; + + freeParam(¶m); + + for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { + SColumnInfo* column = pQueryMsg->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + + //pQInfo already freed in initQInfo, but *pQInfo may not pointer to null; + if (code != TSDB_CODE_SUCCESS) { + *pQInfo = NULL; + } + + // if failed to add ref for all tables in this query, abort current query + return code; +} + +bool qTableQuery(qinfo_t qinfo, uint64_t *qId) { + SQInfo *pQInfo = (SQInfo *)qinfo; + assert(pQInfo && pQInfo->signature == pQInfo); + int64_t threadId = taosGetSelfPthreadId(); + + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { + qError("QInfo:%"PRIu64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); + pQInfo->code = TSDB_CODE_QRY_IN_EXEC; + return false; + } + + + *qId = pQInfo->qId; + pQInfo->startExecTs = taosGetTimestampSec(); + + if (isQueryKilled(pQInfo)) { + qDebug("QInfo:%"PRIu64" it is already killed, abort", pQInfo->qId); + return doBuildResCheck(pQInfo); + } + + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { + qDebug("QInfo:%"PRIu64" no table exists for query, abort", pQInfo->qId); + setQueryStatus(pRuntimeEnv, QUERY_COMPLETED); + return doBuildResCheck(pQInfo); + } + + // error occurs, record the error code and return to client + int32_t ret = setjmp(pQInfo->runtimeEnv.env); + if (ret != TSDB_CODE_SUCCESS) { + pQInfo->code = ret; + qDebug("QInfo:%"PRIu64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); + return doBuildResCheck(pQInfo); + } + + qDebug("QInfo:%"PRIu64" query task is launched", pQInfo->qId); + + pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot); + + if (isQueryKilled(pQInfo)) { + qDebug("QInfo:%"PRIu64" query is killed", pQInfo->qId); + } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { + qDebug("QInfo:%"PRIu64" over, %u tables queried, %"PRId64" rows are returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, + pRuntimeEnv->resultInfo.total); + } else { + qDebug("QInfo:%"PRIu64" query paused, %d rows returned, numOfTotal:%" PRId64 " rows", + pQInfo->qId, GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total + GET_NUM_OF_RESULTS(pRuntimeEnv)); + } + + return doBuildResCheck(pQInfo); +} + +int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + qError("QInfo:%"PRIu64" invalid qhandle", pQInfo->qId); + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + *buildRes = false; + if (IS_QUERY_KILLED(pQInfo)) { + qDebug("QInfo:%"PRIu64" query is killed, code:0x%08x", pQInfo->qId, pQInfo->code); + return pQInfo->code; + } + + int32_t code = TSDB_CODE_SUCCESS; + + if (tsRetrieveBlockingModel) { + pQInfo->rspContext = pRspContext; + tsem_wait(&pQInfo->ready); + *buildRes = true; + code = pQInfo->code; + } else { + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + + pthread_mutex_lock(&pQInfo->lock); + + assert(pQInfo->rspContext == NULL); + if (pQInfo->dataReady == QUERY_RESULT_READY) { + *buildRes = true; + qDebug("QInfo:%"PRIu64" retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo->qId, pQuery->resultRowSize, + GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code)); + } else { + *buildRes = false; + qDebug("QInfo:%"PRIu64" retrieve req set query return result after paused", pQInfo->qId); + pQInfo->rspContext = pRspContext; + assert(pQInfo->rspContext != NULL); + } + + code = pQInfo->code; + pthread_mutex_unlock(&pQInfo->lock); + } + + return code; +} + +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; + + int32_t s = GET_NUM_OF_RESULTS(pRuntimeEnv); + size_t size = pQuery->resultRowSize * s; + size += sizeof(int32_t); + size += sizeof(STableIdInfo) * taosHashGetSize(pRuntimeEnv->pTableRetrieveTsMap); + + *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); + + // current solution only avoid crash, but cannot return error code to client + *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); + if (*pRsp == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + (*pRsp)->numOfRows = htonl((int32_t)s); + + if (pQInfo->code == TSDB_CODE_SUCCESS) { + (*pRsp)->offset = htobe64(pQInfo->runtimeEnv.currentOffset); + (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime); + } else { + (*pRsp)->offset = 0; + (*pRsp)->useconds = htobe64(pQInfo->summary.elapsedTime); + } + + (*pRsp)->precision = htons(pQuery->precision); + if (GET_NUM_OF_RESULTS(&(pQInfo->runtimeEnv)) > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { + doDumpQueryResult(pQInfo, (*pRsp)->data); + } else { + setQueryStatus(pRuntimeEnv, QUERY_OVER); + } + + pQInfo->rspContext = NULL; + pQInfo->dataReady = QUERY_RESULT_NOT_READY; + + if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_OVER)) { + // here current thread hold the refcount, so it is safe to free tsdbQueryHandle. + *continueExec = false; + (*pRsp)->completed = 1; // notify no more result to client + qDebug("QInfo:%"PRIu64" no more results to retrieve", pQInfo->qId); + } else { + *continueExec = true; + qDebug("QInfo:%"PRIu64" has more results to retrieve", pQInfo->qId); + } + + // the memory should be freed if the code of pQInfo is not TSDB_CODE_SUCCESS + if (pQInfo->code != TSDB_CODE_SUCCESS) { + rpcFreeCont(*pRsp); + *pRsp = NULL; + } + + return pQInfo->code; +} + +void* qGetResultRetrieveMsg(qinfo_t qinfo) { + SQInfo* pQInfo = (SQInfo*) qinfo; + assert(pQInfo != NULL); + + return pQInfo->rspContext; +} + +int32_t qKillQuery(qinfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + qDebug("QInfo:%"PRIu64" query killed", pQInfo->qId); + setQueryKilled(pQInfo); + + // Wait for the query executing thread being stopped/ + // Once the query is stopped, the owner of qHandle will be cleared immediately. + while (pQInfo->owner != 0) { + taosMsleep(100); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t qQueryCompleted(qinfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); +} + +void qDestroyQueryInfo(qinfo_t qHandle) { + SQInfo* pQInfo = (SQInfo*) qHandle; + if (!isValidQInfo(pQInfo)) { + return; + } + + qDebug("QInfo:%"PRIu64" query completed", pQInfo->qId); + queryCostStatis(pQInfo); // print the query cost summary + freeQInfo(pQInfo); +} + +void* qOpenQueryMgmt(int32_t vgId) { + const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool + + char cacheName[128] = {0}; + sprintf(cacheName, "qhandle_%d", vgId); + + SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); + if (pQueryMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pQueryMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName); + pQueryMgmt->closed = false; + pQueryMgmt->vgId = vgId; + + pthread_mutex_init(&pQueryMgmt->lock, NULL); + + qDebug("vgId:%d, open querymgmt success", vgId); + return pQueryMgmt; +} + +void qQueryMgmtNotifyClosed(void* pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + SQueryMgmt* pQueryMgmt = pQMgmt; + qInfo("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); + + pthread_mutex_lock(&pQueryMgmt->lock); + pQueryMgmt->closed = true; + pthread_mutex_unlock(&pQueryMgmt->lock); + + taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); +} + +void qQueryMgmtReOpen(void *pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + SQueryMgmt *pQueryMgmt = pQMgmt; + qInfo("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId); + + pthread_mutex_lock(&pQueryMgmt->lock); + pQueryMgmt->closed = false; + pthread_mutex_unlock(&pQueryMgmt->lock); +} + +void qCleanupQueryMgmt(void* pQMgmt) { + if (pQMgmt == NULL) { + return; + } + + SQueryMgmt* pQueryMgmt = pQMgmt; + int32_t vgId = pQueryMgmt->vgId; + + assert(pQueryMgmt->closed); + + SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; + pQueryMgmt->qinfoPool = NULL; + + taosCacheCleanup(pqinfoPool); + pthread_mutex_destroy(&pQueryMgmt->lock); + tfree(pQueryMgmt); + + qDebug("vgId:%d, queryMgmt cleanup completed", vgId); +} + +void** qRegisterQInfo(void* pMgmt, uint64_t qId, uint64_t qInfo) { + if (pMgmt == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + SQueryMgmt *pQueryMgmt = pMgmt; + if (pQueryMgmt->qinfoPool == NULL) { + qError("QInfo:%"PRIu64"-%p failed to add qhandle into qMgmt, since qMgmt is closed", qId, (void*)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + pthread_mutex_lock(&pQueryMgmt->lock); + if (pQueryMgmt->closed) { + pthread_mutex_unlock(&pQueryMgmt->lock); + qError("QInfo:%"PRIu64"-%p failed to add qhandle into cache, since qMgmt is colsing", qId, (void*)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } else { + void** handle = taosCachePut(pQueryMgmt->qinfoPool, &qId, sizeof(qId), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), + (getMaximumIdleDurationSec()*1000)); + pthread_mutex_unlock(&pQueryMgmt->lock); + + return handle; + } +} + +void** qAcquireQInfo(void* pMgmt, uint64_t _key) { + SQueryMgmt *pQueryMgmt = pMgmt; + + if (pQueryMgmt->closed) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; + return NULL; + } + + if (pQueryMgmt->qinfoPool == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; + return NULL; + } + + TSDB_CACHE_PTR_TYPE key = (TSDB_CACHE_PTR_TYPE)_key; + void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &key, sizeof(TSDB_CACHE_PTR_TYPE)); + if (handle == NULL || *handle == NULL) { + terrno = TSDB_CODE_QRY_INVALID_QHANDLE; + return NULL; + } else { + return handle; + } +} + +void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { + SQueryMgmt *pQueryMgmt = pMgmt; + if (pQueryMgmt->qinfoPool == NULL) { + return NULL; + } + + taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); + return 0; +} diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 6c59a5107404eac92632ebcd4f418ccc8468dafe..28208e532af4c4d7688208c19622a352076e9bc8 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -97,27 +97,28 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 279 +#define YYNOCODE 264 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SCreateTableSQL* yy38; - SCreateAcctSQL yy71; - tSQLExpr* yy78; - int yy96; - SQuerySQL* yy148; - SCreatedTableInfo yy152; - SSubclauseInfo* yy153; - tSQLExprList* yy166; - SLimitVal yy167; - TAOS_FIELD yy183; - SCreateDBInfo yy234; - int64_t yy325; - SIntervalVal yy400; - SArray* yy421; - tVariant yy430; + SCreateTableSql* yy14; + int yy20; + tSqlExpr* yy118; + SArray* yy159; + SIntervalVal yy184; + SCreatedTableInfo yy206; + SSessionWindowVal yy249; + SQuerySqlNode* yy272; + int64_t yy317; + SCreateDbInfo yy322; + SCreateAcctInfo yy351; + SSubclauseInfo* yy391; + TAOS_FIELD yy407; + SLimitVal yy440; + tVariant yy488; + SFromInfo* yy514; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -127,17 +128,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 258 -#define YYNRULE 240 -#define YYNTOKEN 209 -#define YY_MAX_SHIFT 257 -#define YY_MIN_SHIFTREDUCE 431 -#define YY_MAX_SHIFTREDUCE 670 -#define YY_ERROR_ACTION 671 -#define YY_ACCEPT_ACTION 672 -#define YY_NO_ACTION 673 -#define YY_MIN_REDUCE 674 -#define YY_MAX_REDUCE 913 +#define YYNSTATE 315 +#define YYNRULE 267 +#define YYNTOKEN 187 +#define YY_MAX_SHIFT 314 +#define YY_MIN_SHIFTREDUCE 506 +#define YY_MAX_SHIFTREDUCE 772 +#define YY_ERROR_ACTION 773 +#define YY_ACCEPT_ACTION 774 +#define YY_NO_ACTION 775 +#define YY_MIN_REDUCE 776 +#define YY_MAX_REDUCE 1042 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -203,228 +204,259 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (586) +#define YY_ACTTAB_COUNT (680) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 143, 474, 143, 23, 672, 257, 165, 547, 827, 475, - /* 10 */ 900, 168, 901, 37, 38, 12, 39, 40, 816, 23, - /* 20 */ 173, 31, 474, 474, 210, 43, 41, 45, 42, 805, - /* 30 */ 475, 475, 163, 36, 35, 232, 231, 34, 33, 32, - /* 40 */ 37, 38, 801, 39, 40, 816, 110, 173, 31, 162, - /* 50 */ 255, 210, 43, 41, 45, 42, 176, 66, 802, 195, - /* 60 */ 36, 35, 178, 824, 34, 33, 32, 432, 433, 434, - /* 70 */ 435, 436, 437, 438, 439, 440, 441, 442, 443, 256, - /* 80 */ 179, 225, 185, 37, 38, 805, 39, 40, 796, 242, - /* 90 */ 173, 31, 143, 180, 210, 43, 41, 45, 42, 110, - /* 100 */ 110, 167, 901, 36, 35, 57, 853, 34, 33, 32, - /* 110 */ 17, 223, 250, 249, 222, 221, 220, 248, 219, 247, - /* 120 */ 246, 245, 218, 244, 243, 803, 142, 774, 624, 762, - /* 130 */ 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, - /* 140 */ 773, 775, 776, 38, 181, 39, 40, 229, 228, 173, - /* 150 */ 31, 605, 606, 210, 43, 41, 45, 42, 207, 854, - /* 160 */ 61, 205, 36, 35, 23, 110, 34, 33, 32, 188, - /* 170 */ 39, 40, 23, 251, 173, 31, 192, 191, 210, 43, - /* 180 */ 41, 45, 42, 34, 33, 32, 105, 36, 35, 104, - /* 190 */ 147, 34, 33, 32, 172, 637, 805, 28, 628, 897, - /* 200 */ 631, 177, 634, 802, 172, 637, 896, 13, 628, 230, - /* 210 */ 631, 802, 634, 18, 172, 637, 794, 895, 628, 63, - /* 220 */ 631, 28, 634, 155, 574, 62, 169, 170, 23, 156, - /* 230 */ 209, 29, 197, 92, 91, 150, 169, 170, 77, 76, - /* 240 */ 582, 17, 198, 250, 249, 159, 169, 170, 248, 626, - /* 250 */ 247, 246, 245, 715, 244, 243, 133, 211, 780, 80, - /* 260 */ 160, 778, 779, 18, 242, 234, 781, 802, 783, 784, - /* 270 */ 782, 28, 785, 786, 43, 41, 45, 42, 724, 579, - /* 280 */ 64, 133, 36, 35, 19, 627, 34, 33, 32, 3, - /* 290 */ 124, 194, 225, 44, 910, 72, 68, 71, 158, 11, - /* 300 */ 10, 566, 592, 44, 563, 636, 564, 107, 565, 793, - /* 310 */ 22, 795, 630, 44, 633, 636, 716, 36, 35, 133, - /* 320 */ 635, 34, 33, 32, 171, 636, 596, 49, 78, 82, - /* 330 */ 635, 48, 182, 183, 87, 90, 81, 137, 135, 629, - /* 340 */ 635, 632, 84, 95, 94, 93, 50, 9, 145, 640, - /* 350 */ 52, 65, 120, 254, 253, 98, 597, 656, 638, 555, - /* 360 */ 146, 15, 14, 14, 24, 4, 55, 53, 546, 213, - /* 370 */ 556, 570, 148, 571, 24, 48, 568, 149, 569, 89, - /* 380 */ 88, 103, 101, 153, 154, 152, 141, 151, 144, 804, - /* 390 */ 864, 863, 818, 174, 860, 859, 175, 233, 826, 846, - /* 400 */ 831, 833, 106, 121, 845, 122, 567, 119, 123, 726, - /* 410 */ 217, 139, 26, 226, 102, 723, 28, 227, 909, 74, - /* 420 */ 908, 906, 125, 744, 27, 25, 196, 140, 713, 591, - /* 430 */ 83, 711, 85, 86, 199, 709, 708, 184, 54, 134, - /* 440 */ 706, 164, 705, 704, 703, 702, 136, 700, 698, 696, - /* 450 */ 694, 692, 138, 203, 58, 59, 51, 847, 815, 46, - /* 460 */ 208, 206, 204, 202, 200, 30, 79, 235, 236, 237, - /* 470 */ 238, 239, 240, 241, 161, 215, 216, 252, 670, 187, - /* 480 */ 186, 669, 69, 189, 157, 190, 668, 193, 661, 707, - /* 490 */ 197, 576, 60, 56, 593, 96, 128, 97, 127, 745, - /* 500 */ 126, 130, 129, 131, 132, 701, 693, 113, 111, 118, - /* 510 */ 116, 114, 112, 115, 800, 1, 117, 2, 166, 20, - /* 520 */ 108, 201, 6, 598, 109, 7, 639, 5, 8, 21, - /* 530 */ 16, 67, 212, 641, 214, 515, 65, 511, 509, 508, - /* 540 */ 507, 504, 478, 224, 70, 47, 73, 75, 24, 549, - /* 550 */ 548, 545, 499, 497, 489, 495, 491, 493, 487, 485, - /* 560 */ 517, 516, 514, 513, 512, 510, 506, 505, 48, 476, - /* 570 */ 447, 445, 674, 673, 673, 673, 673, 673, 673, 673, - /* 580 */ 673, 673, 673, 673, 99, 100, + /* 0 */ 133, 553, 202, 312, 206, 140, 943, 226, 140, 554, + /* 10 */ 774, 314, 17, 47, 48, 140, 51, 52, 30, 181, + /* 20 */ 214, 41, 181, 50, 262, 55, 53, 57, 54, 1023, + /* 30 */ 922, 209, 1024, 46, 45, 179, 181, 44, 43, 42, + /* 40 */ 47, 48, 920, 51, 52, 208, 1024, 214, 41, 553, + /* 50 */ 50, 262, 55, 53, 57, 54, 934, 554, 185, 203, + /* 60 */ 46, 45, 919, 247, 44, 43, 42, 48, 940, 51, + /* 70 */ 52, 242, 974, 214, 41, 79, 50, 262, 55, 53, + /* 80 */ 57, 54, 975, 632, 257, 30, 46, 45, 278, 225, + /* 90 */ 44, 43, 42, 507, 508, 509, 510, 511, 512, 513, + /* 100 */ 514, 515, 516, 517, 518, 519, 313, 553, 85, 231, + /* 110 */ 70, 288, 287, 47, 48, 554, 51, 52, 298, 219, + /* 120 */ 214, 41, 553, 50, 262, 55, 53, 57, 54, 918, + /* 130 */ 554, 105, 718, 46, 45, 1020, 298, 44, 43, 42, + /* 140 */ 47, 49, 910, 51, 52, 922, 140, 214, 41, 234, + /* 150 */ 50, 262, 55, 53, 57, 54, 1019, 238, 237, 227, + /* 160 */ 46, 45, 285, 284, 44, 43, 42, 23, 276, 307, + /* 170 */ 306, 275, 274, 273, 305, 272, 304, 303, 302, 271, + /* 180 */ 301, 300, 882, 30, 870, 871, 872, 873, 874, 875, + /* 190 */ 876, 877, 878, 879, 880, 881, 883, 884, 51, 52, + /* 200 */ 821, 1018, 214, 41, 166, 50, 262, 55, 53, 57, + /* 210 */ 54, 259, 18, 78, 82, 46, 45, 198, 223, 44, + /* 220 */ 43, 42, 213, 731, 217, 25, 722, 919, 725, 190, + /* 230 */ 728, 221, 213, 731, 199, 191, 722, 724, 725, 727, + /* 240 */ 728, 118, 117, 189, 263, 905, 906, 29, 909, 44, + /* 250 */ 43, 42, 30, 74, 210, 211, 308, 922, 261, 30, + /* 260 */ 23, 36, 307, 306, 210, 211, 934, 305, 30, 304, + /* 270 */ 303, 302, 74, 301, 300, 890, 908, 183, 888, 889, + /* 280 */ 36, 204, 922, 891, 916, 893, 894, 892, 224, 895, + /* 290 */ 896, 280, 656, 218, 830, 653, 919, 654, 166, 655, + /* 300 */ 281, 69, 241, 919, 68, 55, 53, 57, 54, 282, + /* 310 */ 197, 671, 919, 46, 45, 30, 822, 44, 43, 42, + /* 320 */ 166, 103, 108, 228, 229, 56, 220, 97, 107, 113, + /* 330 */ 116, 106, 732, 907, 723, 56, 726, 110, 730, 30, + /* 340 */ 735, 12, 732, 5, 156, 84, 184, 81, 730, 33, + /* 350 */ 155, 92, 87, 91, 729, 278, 286, 1, 154, 919, + /* 360 */ 174, 170, 186, 212, 729, 80, 172, 169, 121, 120, + /* 370 */ 119, 46, 45, 3, 167, 44, 43, 42, 71, 720, + /* 380 */ 290, 699, 700, 919, 311, 310, 126, 243, 668, 675, + /* 390 */ 678, 31, 684, 690, 180, 24, 135, 60, 245, 691, + /* 400 */ 752, 657, 733, 20, 19, 61, 19, 6, 64, 642, + /* 410 */ 265, 1034, 644, 31, 31, 721, 60, 267, 643, 187, + /* 420 */ 28, 83, 60, 268, 96, 95, 188, 62, 65, 14, + /* 430 */ 13, 102, 101, 660, 67, 661, 631, 194, 16, 15, + /* 440 */ 658, 195, 659, 115, 114, 131, 129, 193, 178, 192, + /* 450 */ 182, 921, 985, 984, 215, 981, 239, 980, 132, 942, + /* 460 */ 216, 289, 39, 950, 952, 967, 134, 966, 138, 935, + /* 470 */ 246, 130, 248, 917, 151, 915, 150, 205, 683, 250, + /* 480 */ 152, 886, 299, 153, 291, 148, 146, 260, 142, 932, + /* 490 */ 141, 58, 833, 270, 66, 255, 143, 37, 63, 176, + /* 500 */ 34, 279, 829, 258, 144, 1039, 93, 256, 1038, 1036, + /* 510 */ 157, 254, 283, 1033, 99, 1032, 1030, 158, 851, 35, + /* 520 */ 252, 145, 32, 38, 177, 818, 109, 816, 111, 40, + /* 530 */ 112, 814, 813, 230, 168, 811, 810, 809, 808, 807, + /* 540 */ 806, 171, 173, 803, 801, 799, 797, 795, 175, 249, + /* 550 */ 244, 72, 75, 104, 251, 968, 292, 293, 294, 295, + /* 560 */ 296, 297, 309, 200, 222, 269, 772, 232, 201, 233, + /* 570 */ 771, 88, 89, 196, 235, 236, 770, 758, 757, 240, + /* 580 */ 245, 8, 264, 73, 812, 663, 805, 161, 852, 159, + /* 590 */ 160, 163, 162, 164, 165, 122, 123, 124, 804, 76, + /* 600 */ 125, 796, 4, 2, 685, 136, 137, 688, 77, 149, + /* 610 */ 147, 207, 253, 86, 692, 898, 139, 9, 10, 26, + /* 620 */ 27, 734, 7, 11, 736, 21, 22, 266, 595, 591, + /* 630 */ 589, 84, 588, 587, 584, 557, 277, 94, 90, 31, + /* 640 */ 634, 633, 59, 630, 579, 577, 98, 569, 575, 571, + /* 650 */ 573, 567, 565, 100, 598, 597, 596, 594, 593, 592, + /* 660 */ 590, 586, 585, 60, 555, 523, 521, 776, 775, 127, + /* 670 */ 775, 775, 775, 775, 775, 775, 775, 775, 775, 128, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 267, 1, 267, 213, 210, 211, 230, 5, 213, 9, - /* 10 */ 277, 276, 277, 13, 14, 267, 16, 17, 251, 213, - /* 20 */ 20, 21, 1, 1, 24, 25, 26, 27, 28, 253, - /* 30 */ 9, 9, 265, 33, 34, 33, 34, 37, 38, 39, - /* 40 */ 13, 14, 252, 16, 17, 251, 213, 20, 21, 212, - /* 50 */ 213, 24, 25, 26, 27, 28, 250, 218, 252, 265, - /* 60 */ 33, 34, 230, 268, 37, 38, 39, 45, 46, 47, - /* 70 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 80 */ 66, 76, 60, 13, 14, 253, 16, 17, 249, 78, - /* 90 */ 20, 21, 267, 213, 24, 25, 26, 27, 28, 213, - /* 100 */ 213, 276, 277, 33, 34, 105, 273, 37, 38, 39, - /* 110 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 120 */ 95, 96, 97, 98, 99, 245, 267, 229, 101, 231, - /* 130 */ 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, - /* 140 */ 242, 243, 244, 14, 130, 16, 17, 133, 134, 20, - /* 150 */ 21, 118, 119, 24, 25, 26, 27, 28, 271, 273, - /* 160 */ 273, 275, 33, 34, 213, 213, 37, 38, 39, 129, - /* 170 */ 16, 17, 213, 230, 20, 21, 136, 137, 24, 25, - /* 180 */ 26, 27, 28, 37, 38, 39, 213, 33, 34, 100, - /* 190 */ 267, 37, 38, 39, 1, 2, 253, 108, 5, 267, - /* 200 */ 7, 250, 9, 252, 1, 2, 267, 44, 5, 250, - /* 210 */ 7, 252, 9, 100, 1, 2, 0, 267, 5, 254, - /* 220 */ 7, 108, 9, 60, 101, 273, 33, 34, 213, 66, - /* 230 */ 37, 266, 109, 70, 71, 72, 33, 34, 131, 132, - /* 240 */ 37, 85, 269, 87, 88, 267, 33, 34, 92, 1, - /* 250 */ 94, 95, 96, 217, 98, 99, 220, 15, 229, 73, - /* 260 */ 267, 232, 233, 100, 78, 250, 237, 252, 239, 240, - /* 270 */ 241, 108, 243, 244, 25, 26, 27, 28, 217, 106, - /* 280 */ 218, 220, 33, 34, 111, 37, 37, 38, 39, 61, - /* 290 */ 62, 128, 76, 100, 253, 67, 68, 69, 135, 131, - /* 300 */ 132, 2, 101, 100, 5, 112, 7, 106, 9, 247, - /* 310 */ 248, 249, 5, 100, 7, 112, 217, 33, 34, 220, - /* 320 */ 127, 37, 38, 39, 59, 112, 101, 106, 61, 62, - /* 330 */ 127, 106, 33, 34, 67, 68, 69, 61, 62, 5, - /* 340 */ 127, 7, 75, 67, 68, 69, 125, 100, 267, 107, - /* 350 */ 106, 104, 105, 63, 64, 65, 101, 101, 101, 101, - /* 360 */ 267, 106, 106, 106, 106, 100, 100, 123, 102, 101, - /* 370 */ 101, 5, 267, 7, 106, 106, 5, 267, 7, 73, - /* 380 */ 74, 61, 62, 267, 267, 267, 267, 267, 267, 253, - /* 390 */ 246, 246, 251, 246, 246, 246, 246, 246, 213, 274, - /* 400 */ 213, 213, 213, 213, 274, 213, 107, 255, 213, 213, - /* 410 */ 213, 213, 213, 213, 59, 213, 108, 213, 213, 213, - /* 420 */ 213, 213, 213, 213, 213, 213, 251, 213, 213, 112, - /* 430 */ 213, 213, 213, 213, 270, 213, 213, 213, 122, 213, - /* 440 */ 213, 270, 213, 213, 213, 213, 213, 213, 213, 213, - /* 450 */ 213, 213, 213, 270, 214, 214, 124, 214, 264, 121, - /* 460 */ 116, 120, 115, 114, 113, 126, 84, 83, 49, 80, - /* 470 */ 82, 53, 81, 79, 214, 214, 214, 76, 5, 5, - /* 480 */ 138, 5, 218, 138, 214, 5, 5, 129, 86, 214, - /* 490 */ 109, 101, 106, 110, 101, 215, 222, 215, 226, 228, - /* 500 */ 227, 223, 225, 224, 221, 214, 214, 261, 263, 256, - /* 510 */ 258, 260, 262, 259, 251, 219, 257, 216, 1, 106, - /* 520 */ 100, 100, 117, 101, 100, 117, 101, 100, 100, 106, - /* 530 */ 100, 73, 103, 107, 103, 9, 104, 5, 5, 5, - /* 540 */ 5, 5, 77, 15, 73, 16, 132, 132, 106, 5, - /* 550 */ 5, 101, 5, 5, 5, 5, 5, 5, 5, 5, - /* 560 */ 5, 5, 5, 5, 5, 5, 5, 5, 106, 77, - /* 570 */ 59, 58, 0, 278, 278, 278, 278, 278, 278, 278, - /* 580 */ 278, 278, 278, 278, 21, 21, 278, 278, 278, 278, - /* 590 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 600 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 610 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 620 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 630 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 640 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 650 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 660 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 670 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 680 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 690 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 700 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 710 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 720 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 730 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 740 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 750 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 760 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 770 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 780 */ 278, 278, 278, 278, 278, 278, 278, 278, 278, 278, - /* 790 */ 278, 278, 278, 278, 278, + /* 0 */ 191, 1, 190, 191, 210, 191, 191, 191, 191, 9, + /* 10 */ 188, 189, 252, 13, 14, 191, 16, 17, 191, 252, + /* 20 */ 20, 21, 252, 23, 24, 25, 26, 27, 28, 262, + /* 30 */ 236, 261, 262, 33, 34, 252, 252, 37, 38, 39, + /* 40 */ 13, 14, 226, 16, 17, 261, 262, 20, 21, 1, + /* 50 */ 23, 24, 25, 26, 27, 28, 234, 9, 252, 232, + /* 60 */ 33, 34, 235, 254, 37, 38, 39, 14, 253, 16, + /* 70 */ 17, 249, 258, 20, 21, 258, 23, 24, 25, 26, + /* 80 */ 27, 28, 258, 5, 260, 191, 33, 34, 79, 67, + /* 90 */ 37, 38, 39, 45, 46, 47, 48, 49, 50, 51, + /* 100 */ 52, 53, 54, 55, 56, 57, 58, 1, 197, 61, + /* 110 */ 110, 33, 34, 13, 14, 9, 16, 17, 81, 210, + /* 120 */ 20, 21, 1, 23, 24, 25, 26, 27, 28, 235, + /* 130 */ 9, 76, 105, 33, 34, 252, 81, 37, 38, 39, + /* 140 */ 13, 14, 231, 16, 17, 236, 191, 20, 21, 135, + /* 150 */ 23, 24, 25, 26, 27, 28, 252, 143, 144, 137, + /* 160 */ 33, 34, 140, 141, 37, 38, 39, 88, 89, 90, + /* 170 */ 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, + /* 180 */ 101, 102, 209, 191, 211, 212, 213, 214, 215, 216, + /* 190 */ 217, 218, 219, 220, 221, 222, 223, 224, 16, 17, + /* 200 */ 196, 252, 20, 21, 200, 23, 24, 25, 26, 27, + /* 210 */ 28, 256, 44, 258, 197, 33, 34, 252, 67, 37, + /* 220 */ 38, 39, 1, 2, 232, 104, 5, 235, 7, 61, + /* 230 */ 9, 210, 1, 2, 252, 67, 5, 5, 7, 7, + /* 240 */ 9, 73, 74, 75, 15, 228, 229, 230, 231, 37, + /* 250 */ 38, 39, 191, 104, 33, 34, 210, 236, 37, 191, + /* 260 */ 88, 112, 90, 91, 33, 34, 234, 95, 191, 97, + /* 270 */ 98, 99, 104, 101, 102, 209, 0, 252, 212, 213, + /* 280 */ 112, 249, 236, 217, 191, 219, 220, 221, 137, 223, + /* 290 */ 224, 140, 2, 232, 196, 5, 235, 7, 200, 9, + /* 300 */ 232, 197, 134, 235, 136, 25, 26, 27, 28, 232, + /* 310 */ 142, 37, 235, 33, 34, 191, 196, 37, 38, 39, + /* 320 */ 200, 62, 63, 33, 34, 104, 233, 68, 69, 70, + /* 330 */ 71, 72, 111, 229, 5, 104, 7, 78, 117, 191, + /* 340 */ 111, 104, 111, 62, 63, 108, 252, 110, 117, 68, + /* 350 */ 69, 70, 71, 72, 133, 79, 232, 198, 199, 235, + /* 360 */ 62, 63, 252, 60, 133, 237, 68, 69, 70, 71, + /* 370 */ 72, 33, 34, 194, 195, 37, 38, 39, 250, 1, + /* 380 */ 232, 124, 125, 235, 64, 65, 66, 105, 109, 115, + /* 390 */ 105, 109, 105, 105, 252, 116, 109, 109, 113, 105, + /* 400 */ 105, 111, 105, 109, 109, 109, 109, 104, 109, 105, + /* 410 */ 105, 236, 105, 109, 109, 37, 109, 105, 105, 252, + /* 420 */ 104, 109, 109, 107, 138, 139, 252, 131, 129, 138, + /* 430 */ 139, 138, 139, 5, 104, 7, 106, 252, 138, 139, + /* 440 */ 5, 252, 7, 76, 77, 62, 63, 252, 252, 252, + /* 450 */ 252, 236, 227, 227, 227, 227, 191, 227, 191, 191, + /* 460 */ 227, 227, 251, 191, 191, 259, 191, 259, 191, 234, + /* 470 */ 234, 60, 255, 234, 191, 191, 238, 255, 117, 255, + /* 480 */ 191, 225, 103, 191, 86, 240, 242, 122, 246, 248, + /* 490 */ 247, 127, 191, 191, 128, 255, 245, 191, 130, 191, + /* 500 */ 191, 191, 191, 126, 244, 191, 191, 121, 191, 191, + /* 510 */ 191, 120, 191, 191, 191, 191, 191, 191, 191, 191, + /* 520 */ 119, 243, 191, 191, 191, 191, 191, 191, 191, 132, + /* 530 */ 191, 191, 191, 191, 191, 191, 191, 191, 191, 191, + /* 540 */ 191, 191, 191, 191, 191, 191, 191, 191, 191, 118, + /* 550 */ 192, 192, 192, 87, 192, 192, 50, 83, 85, 54, + /* 560 */ 84, 82, 79, 192, 192, 192, 5, 145, 192, 5, + /* 570 */ 5, 197, 197, 192, 145, 5, 5, 90, 89, 135, + /* 580 */ 113, 104, 107, 114, 192, 105, 192, 202, 208, 207, + /* 590 */ 206, 203, 205, 204, 201, 193, 193, 193, 192, 109, + /* 600 */ 193, 192, 194, 198, 105, 104, 109, 105, 104, 239, + /* 610 */ 241, 1, 104, 76, 105, 225, 104, 123, 123, 109, + /* 620 */ 109, 105, 104, 104, 111, 104, 104, 107, 9, 5, + /* 630 */ 5, 108, 5, 5, 5, 80, 15, 139, 76, 109, + /* 640 */ 5, 5, 16, 105, 5, 5, 139, 5, 5, 5, + /* 650 */ 5, 5, 5, 139, 5, 5, 5, 5, 5, 5, + /* 660 */ 5, 5, 5, 109, 80, 60, 59, 0, 263, 21, + /* 670 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 21, + /* 680 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 690 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 700 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 710 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 720 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 730 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 740 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 750 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 760 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 770 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 780 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 790 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 800 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 810 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 820 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 830 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 840 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 850 */ 263, 263, 263, 263, 263, 263, 263, 263, 263, 263, + /* 860 */ 263, 263, 263, 263, 263, 263, 263, }; -#define YY_SHIFT_COUNT (257) +#define YY_SHIFT_COUNT (314) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (572) +#define YY_SHIFT_MAX (667) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 163, 25, 156, 5, 193, 213, 21, 21, 21, 21, - /* 10 */ 21, 21, 0, 22, 213, 299, 299, 299, 113, 21, - /* 20 */ 21, 21, 216, 21, 21, 186, 11, 11, 586, 203, - /* 30 */ 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - /* 40 */ 213, 213, 213, 213, 213, 213, 213, 299, 299, 2, - /* 50 */ 2, 2, 2, 2, 2, 2, 89, 21, 21, 21, - /* 60 */ 21, 33, 33, 173, 21, 21, 21, 21, 21, 21, - /* 70 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - /* 80 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - /* 90 */ 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, - /* 100 */ 21, 21, 21, 21, 308, 355, 355, 317, 317, 317, - /* 110 */ 355, 316, 332, 338, 344, 341, 347, 349, 351, 339, - /* 120 */ 308, 355, 355, 355, 5, 355, 382, 384, 419, 389, - /* 130 */ 388, 418, 391, 394, 355, 401, 355, 401, 355, 586, - /* 140 */ 586, 27, 70, 70, 70, 129, 154, 249, 249, 249, - /* 150 */ 267, 284, 284, 284, 284, 228, 276, 14, 40, 146, - /* 160 */ 146, 247, 290, 123, 201, 225, 255, 256, 257, 307, - /* 170 */ 334, 248, 265, 242, 221, 244, 258, 268, 269, 107, - /* 180 */ 266, 168, 366, 371, 306, 320, 473, 342, 474, 476, - /* 190 */ 345, 480, 481, 402, 358, 381, 390, 383, 386, 393, - /* 200 */ 420, 517, 421, 422, 424, 413, 405, 423, 408, 425, - /* 210 */ 427, 426, 428, 429, 430, 431, 432, 458, 526, 532, - /* 220 */ 533, 534, 535, 536, 465, 528, 471, 529, 414, 415, - /* 230 */ 442, 544, 545, 450, 442, 547, 548, 549, 550, 551, - /* 240 */ 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, - /* 250 */ 562, 462, 492, 563, 564, 511, 513, 572, + /* 0 */ 168, 79, 79, 172, 172, 9, 221, 231, 106, 106, + /* 10 */ 106, 106, 106, 106, 106, 106, 106, 0, 48, 231, + /* 20 */ 290, 290, 290, 290, 121, 149, 106, 106, 106, 276, + /* 30 */ 106, 106, 55, 9, 37, 37, 680, 680, 680, 231, + /* 40 */ 231, 231, 231, 231, 231, 231, 231, 231, 231, 231, + /* 50 */ 231, 231, 231, 231, 231, 231, 231, 231, 231, 290, + /* 60 */ 290, 78, 78, 78, 78, 78, 78, 78, 106, 106, + /* 70 */ 106, 274, 106, 149, 149, 106, 106, 106, 257, 257, + /* 80 */ 279, 149, 106, 106, 106, 106, 106, 106, 106, 106, + /* 90 */ 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, + /* 100 */ 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, + /* 110 */ 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, + /* 120 */ 106, 106, 106, 106, 106, 106, 106, 106, 106, 106, + /* 130 */ 106, 106, 411, 411, 411, 361, 361, 361, 411, 361, + /* 140 */ 411, 366, 368, 364, 365, 377, 386, 391, 401, 431, + /* 150 */ 397, 411, 411, 411, 379, 9, 9, 411, 411, 466, + /* 160 */ 398, 506, 474, 473, 505, 476, 479, 379, 411, 483, + /* 170 */ 483, 411, 483, 411, 483, 411, 680, 680, 27, 100, + /* 180 */ 127, 100, 100, 53, 182, 280, 280, 280, 280, 259, + /* 190 */ 281, 298, 338, 338, 338, 338, 22, 14, 212, 212, + /* 200 */ 237, 151, 320, 282, 285, 287, 288, 294, 295, 297, + /* 210 */ 232, 329, 378, 303, 229, 296, 299, 304, 305, 307, + /* 220 */ 312, 313, 316, 286, 291, 293, 330, 300, 428, 435, + /* 230 */ 367, 383, 561, 422, 564, 565, 429, 570, 571, 487, + /* 240 */ 489, 444, 467, 475, 477, 469, 480, 490, 499, 501, + /* 250 */ 502, 497, 504, 610, 508, 509, 512, 510, 494, 511, + /* 260 */ 495, 516, 518, 513, 519, 475, 521, 520, 522, 523, + /* 270 */ 537, 619, 624, 625, 627, 628, 629, 555, 621, 562, + /* 280 */ 498, 530, 530, 626, 507, 514, 530, 635, 636, 538, + /* 290 */ 530, 639, 640, 642, 643, 644, 645, 646, 647, 649, + /* 300 */ 650, 651, 652, 653, 654, 655, 656, 657, 554, 584, + /* 310 */ 648, 658, 605, 607, 667, }; -#define YY_REDUCE_COUNT (140) -#define YY_REDUCE_MIN (-267) -#define YY_REDUCE_MAX (301) +#define YY_REDUCE_COUNT (177) +#define YY_REDUCE_MIN (-240) +#define YY_REDUCE_MAX (409) static const short yy_reduce_ofst[] = { - /* 0 */ -206, -102, 29, 62, -265, -175, -114, -113, -194, -49, - /* 10 */ -41, 15, -205, -163, -267, -224, -168, -57, -233, -27, - /* 20 */ -167, -48, -161, -120, -210, 36, 61, 99, -35, -252, - /* 30 */ -141, -77, -68, -61, -50, -22, -7, 81, 93, 105, - /* 40 */ 110, 116, 117, 118, 119, 120, 121, 41, 136, 144, - /* 50 */ 145, 147, 148, 149, 150, 151, 141, 185, 187, 188, - /* 60 */ 189, 125, 130, 152, 190, 192, 195, 196, 197, 198, - /* 70 */ 199, 200, 202, 204, 205, 206, 207, 208, 209, 210, - /* 80 */ 211, 212, 214, 215, 217, 218, 219, 220, 222, 223, - /* 90 */ 224, 226, 227, 229, 230, 231, 232, 233, 234, 235, - /* 100 */ 236, 237, 238, 239, 175, 240, 241, 164, 171, 183, - /* 110 */ 243, 194, 245, 250, 246, 251, 254, 252, 259, 253, - /* 120 */ 263, 260, 261, 262, 264, 270, 271, 273, 272, 274, - /* 130 */ 277, 278, 279, 283, 275, 280, 291, 282, 292, 296, - /* 140 */ 301, + /* 0 */ -178, -27, -27, 66, 66, 17, -230, -216, -173, -176, + /* 10 */ -45, -8, 61, 68, 77, 124, 148, -185, -188, -233, + /* 20 */ -206, -91, 21, 46, -191, 32, -186, -183, 93, -89, + /* 30 */ -184, -106, 4, 104, 98, 120, 128, 159, 179, -240, + /* 40 */ -217, -194, -117, -96, -51, -35, -18, 25, 94, 110, + /* 50 */ 142, 167, 174, 185, 189, 195, 196, 197, 198, 175, + /* 60 */ 215, 225, 226, 227, 228, 230, 233, 234, 265, 267, + /* 70 */ 268, 211, 272, 235, 236, 273, 275, 277, 206, 208, + /* 80 */ 238, 239, 283, 284, 289, 292, 301, 302, 306, 308, + /* 90 */ 309, 310, 311, 314, 315, 317, 318, 319, 321, 322, + /* 100 */ 323, 324, 325, 326, 327, 328, 331, 332, 333, 334, + /* 110 */ 335, 336, 337, 339, 340, 341, 342, 343, 344, 345, + /* 120 */ 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, + /* 130 */ 356, 357, 358, 359, 360, 217, 222, 224, 362, 240, + /* 140 */ 363, 241, 243, 242, 251, 260, 278, 244, 369, 245, + /* 150 */ 370, 371, 372, 373, 256, 374, 375, 376, 381, 380, + /* 160 */ 382, 384, 385, 387, 388, 389, 393, 390, 392, 402, + /* 170 */ 403, 394, 404, 406, 407, 409, 405, 408, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 671, 725, 714, 722, 903, 903, 671, 671, 671, 671, - /* 10 */ 671, 671, 828, 689, 903, 671, 671, 671, 671, 671, - /* 20 */ 671, 671, 722, 671, 671, 727, 727, 727, 823, 671, - /* 30 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 40 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 50 */ 671, 671, 671, 671, 671, 671, 671, 671, 830, 832, - /* 60 */ 671, 850, 850, 821, 671, 671, 671, 671, 671, 671, - /* 70 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 80 */ 671, 671, 671, 712, 671, 710, 671, 671, 671, 671, - /* 90 */ 671, 671, 671, 671, 671, 671, 671, 671, 699, 671, - /* 100 */ 671, 671, 671, 671, 671, 691, 691, 671, 671, 671, - /* 110 */ 691, 857, 861, 855, 843, 851, 842, 838, 837, 865, - /* 120 */ 671, 691, 691, 691, 722, 691, 743, 741, 739, 731, - /* 130 */ 737, 733, 735, 729, 691, 720, 691, 720, 691, 761, - /* 140 */ 777, 671, 866, 902, 856, 892, 891, 898, 890, 889, - /* 150 */ 671, 885, 886, 888, 887, 671, 671, 671, 671, 894, - /* 160 */ 893, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 170 */ 671, 671, 868, 671, 862, 858, 671, 671, 671, 671, - /* 180 */ 787, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 190 */ 671, 671, 671, 671, 671, 820, 671, 671, 829, 671, - /* 200 */ 671, 671, 671, 671, 671, 852, 671, 844, 671, 671, - /* 210 */ 671, 671, 671, 797, 671, 671, 671, 671, 671, 671, - /* 220 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 230 */ 907, 671, 671, 671, 905, 671, 671, 671, 671, 671, - /* 240 */ 671, 671, 671, 671, 671, 671, 671, 671, 671, 671, - /* 250 */ 671, 746, 671, 697, 695, 671, 687, 671, + /* 0 */ 773, 885, 831, 897, 819, 828, 1026, 1026, 773, 773, + /* 10 */ 773, 773, 773, 773, 773, 773, 773, 944, 792, 1026, + /* 20 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 828, + /* 30 */ 773, 773, 834, 828, 834, 834, 939, 869, 887, 773, + /* 40 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 50 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 60 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 70 */ 773, 946, 949, 773, 773, 951, 773, 773, 971, 971, + /* 80 */ 937, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 90 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 100 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 817, + /* 110 */ 773, 815, 773, 773, 773, 773, 773, 773, 773, 773, + /* 120 */ 773, 773, 773, 773, 773, 773, 802, 773, 773, 773, + /* 130 */ 773, 773, 794, 794, 794, 773, 773, 773, 794, 773, + /* 140 */ 794, 978, 982, 976, 964, 972, 963, 959, 957, 956, + /* 150 */ 986, 794, 794, 794, 832, 828, 828, 794, 794, 850, + /* 160 */ 848, 846, 838, 844, 840, 842, 836, 820, 794, 826, + /* 170 */ 826, 794, 826, 794, 826, 794, 869, 887, 773, 987, + /* 180 */ 773, 1025, 977, 1015, 1014, 1021, 1013, 1012, 1011, 773, + /* 190 */ 773, 773, 1007, 1008, 1010, 1009, 773, 773, 1017, 1016, + /* 200 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 210 */ 773, 773, 773, 989, 773, 983, 979, 773, 773, 773, + /* 220 */ 773, 773, 773, 773, 773, 773, 899, 773, 773, 773, + /* 230 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 240 */ 773, 773, 936, 773, 773, 773, 773, 947, 773, 773, + /* 250 */ 773, 773, 773, 773, 773, 773, 773, 973, 773, 965, + /* 260 */ 773, 773, 773, 773, 773, 911, 773, 773, 773, 773, + /* 270 */ 773, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 280 */ 773, 1037, 1035, 773, 773, 773, 1031, 773, 773, 773, + /* 290 */ 1029, 773, 773, 773, 773, 773, 773, 773, 773, 773, + /* 300 */ 773, 773, 773, 773, 773, 773, 773, 773, 853, 773, + /* 310 */ 800, 798, 773, 790, 773, }; /********** End of lemon-generated parsing tables *****************************/ @@ -490,6 +522,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* BITNOT => nothing */ 0, /* SHOW => nothing */ 0, /* DATABASES => nothing */ + 0, /* TOPICS => nothing */ 0, /* MNODES => nothing */ 0, /* DNODES => nothing */ 0, /* ACCOUNTS => nothing */ @@ -511,6 +544,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* STABLES => nothing */ 0, /* VGROUPS => nothing */ 0, /* DROP => nothing */ + 1, /* STABLE => ID */ + 0, /* TOPIC => nothing */ 0, /* DNODE => nothing */ 0, /* USER => nothing */ 0, /* ACCOUNT => nothing */ @@ -544,20 +579,23 @@ static const YYCODETYPE yyFallback[] = { 0, /* PRECISION => nothing */ 0, /* UPDATE => nothing */ 0, /* CACHELAST => nothing */ + 0, /* PARTITIONS => nothing */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* UNSIGNED => nothing */ 0, /* TAGS => nothing */ 0, /* USING => nothing */ - 0, /* AS => nothing */ 0, /* COMMA => nothing */ + 0, /* AS => nothing */ 1, /* NULL => ID */ 0, /* SELECT => nothing */ 0, /* UNION => nothing */ 1, /* ALL => ID */ + 0, /* DISTINCT => nothing */ 0, /* FROM => nothing */ 0, /* VARIABLE => nothing */ 0, /* INTERVAL => nothing */ + 0, /* SESSION => nothing */ 0, /* FILL => nothing */ 0, /* SLIDING => nothing */ 0, /* ORDER => nothing */ @@ -574,6 +612,7 @@ static const YYCODETYPE yyFallback[] = { 1, /* NOW => ID */ 0, /* RESET => nothing */ 0, /* QUERY => nothing */ + 0, /* SYNCDB => nothing */ 0, /* ADD => nothing */ 0, /* COLUMN => nothing */ 0, /* TAG => nothing */ @@ -614,42 +653,13 @@ static const YYCODETYPE yyFallback[] = { 1, /* STATEMENT => ID */ 1, /* TRIGGER => ID */ 1, /* VIEW => ID */ - 1, /* COUNT => ID */ - 1, /* SUM => ID */ - 1, /* AVG => ID */ - 1, /* MIN => ID */ - 1, /* MAX => ID */ - 1, /* FIRST => ID */ - 1, /* LAST => ID */ - 1, /* TOP => ID */ - 1, /* BOTTOM => ID */ - 1, /* STDDEV => ID */ - 1, /* PERCENTILE => ID */ - 1, /* APERCENTILE => ID */ - 1, /* LEASTSQUARES => ID */ - 1, /* HISTOGRAM => ID */ - 1, /* DIFF => ID */ - 1, /* SPREAD => ID */ - 1, /* TWA => ID */ - 1, /* INTERP => ID */ - 1, /* LAST_ROW => ID */ - 1, /* RATE => ID */ - 1, /* IRATE => ID */ - 1, /* SUM_RATE => ID */ - 1, /* SUM_IRATE => ID */ - 1, /* AVG_RATE => ID */ - 1, /* AVG_IRATE => ID */ - 1, /* TBID => ID */ 1, /* SEMI => ID */ 1, /* NONE => ID */ 1, /* PREV => ID */ 1, /* LINEAR => ID */ 1, /* IMPORT => ID */ - 1, /* METRIC => ID */ 1, /* TBNAME => ID */ 1, /* JOIN => ID */ - 1, /* METRICS => ID */ - 1, /* STABLE => ID */ 1, /* INSERT => ID */ 1, /* INTO => ID */ 1, /* VALUES => ID */ @@ -785,238 +795,223 @@ static const char *const yyTokenName[] = { /* 43 */ "BITNOT", /* 44 */ "SHOW", /* 45 */ "DATABASES", - /* 46 */ "MNODES", - /* 47 */ "DNODES", - /* 48 */ "ACCOUNTS", - /* 49 */ "USERS", - /* 50 */ "MODULES", - /* 51 */ "QUERIES", - /* 52 */ "CONNECTIONS", - /* 53 */ "STREAMS", - /* 54 */ "VARIABLES", - /* 55 */ "SCORES", - /* 56 */ "GRANTS", - /* 57 */ "VNODES", - /* 58 */ "IPTOKEN", - /* 59 */ "DOT", - /* 60 */ "CREATE", - /* 61 */ "TABLE", - /* 62 */ "DATABASE", - /* 63 */ "TABLES", - /* 64 */ "STABLES", - /* 65 */ "VGROUPS", - /* 66 */ "DROP", - /* 67 */ "DNODE", - /* 68 */ "USER", - /* 69 */ "ACCOUNT", - /* 70 */ "USE", - /* 71 */ "DESCRIBE", - /* 72 */ "ALTER", - /* 73 */ "PASS", - /* 74 */ "PRIVILEGE", - /* 75 */ "LOCAL", - /* 76 */ "IF", - /* 77 */ "EXISTS", - /* 78 */ "PPS", - /* 79 */ "TSERIES", - /* 80 */ "DBS", - /* 81 */ "STORAGE", - /* 82 */ "QTIME", - /* 83 */ "CONNS", - /* 84 */ "STATE", - /* 85 */ "KEEP", - /* 86 */ "CACHE", - /* 87 */ "REPLICA", - /* 88 */ "QUORUM", - /* 89 */ "DAYS", - /* 90 */ "MINROWS", - /* 91 */ "MAXROWS", - /* 92 */ "BLOCKS", - /* 93 */ "CTIME", - /* 94 */ "WAL", - /* 95 */ "FSYNC", - /* 96 */ "COMP", - /* 97 */ "PRECISION", - /* 98 */ "UPDATE", - /* 99 */ "CACHELAST", - /* 100 */ "LP", - /* 101 */ "RP", - /* 102 */ "UNSIGNED", - /* 103 */ "TAGS", - /* 104 */ "USING", - /* 105 */ "AS", - /* 106 */ "COMMA", - /* 107 */ "NULL", - /* 108 */ "SELECT", - /* 109 */ "UNION", - /* 110 */ "ALL", - /* 111 */ "FROM", - /* 112 */ "VARIABLE", - /* 113 */ "INTERVAL", - /* 114 */ "FILL", - /* 115 */ "SLIDING", - /* 116 */ "ORDER", - /* 117 */ "BY", - /* 118 */ "ASC", - /* 119 */ "DESC", - /* 120 */ "GROUP", - /* 121 */ "HAVING", - /* 122 */ "LIMIT", - /* 123 */ "OFFSET", - /* 124 */ "SLIMIT", - /* 125 */ "SOFFSET", - /* 126 */ "WHERE", - /* 127 */ "NOW", - /* 128 */ "RESET", - /* 129 */ "QUERY", - /* 130 */ "ADD", - /* 131 */ "COLUMN", - /* 132 */ "TAG", - /* 133 */ "CHANGE", - /* 134 */ "SET", - /* 135 */ "KILL", - /* 136 */ "CONNECTION", - /* 137 */ "STREAM", - /* 138 */ "COLON", - /* 139 */ "ABORT", - /* 140 */ "AFTER", - /* 141 */ "ATTACH", - /* 142 */ "BEFORE", - /* 143 */ "BEGIN", - /* 144 */ "CASCADE", - /* 145 */ "CLUSTER", - /* 146 */ "CONFLICT", - /* 147 */ "COPY", - /* 148 */ "DEFERRED", - /* 149 */ "DELIMITERS", - /* 150 */ "DETACH", - /* 151 */ "EACH", - /* 152 */ "END", - /* 153 */ "EXPLAIN", - /* 154 */ "FAIL", - /* 155 */ "FOR", - /* 156 */ "IGNORE", - /* 157 */ "IMMEDIATE", - /* 158 */ "INITIALLY", - /* 159 */ "INSTEAD", - /* 160 */ "MATCH", - /* 161 */ "KEY", - /* 162 */ "OF", - /* 163 */ "RAISE", - /* 164 */ "REPLACE", - /* 165 */ "RESTRICT", - /* 166 */ "ROW", - /* 167 */ "STATEMENT", - /* 168 */ "TRIGGER", - /* 169 */ "VIEW", - /* 170 */ "COUNT", - /* 171 */ "SUM", - /* 172 */ "AVG", - /* 173 */ "MIN", - /* 174 */ "MAX", - /* 175 */ "FIRST", - /* 176 */ "LAST", - /* 177 */ "TOP", - /* 178 */ "BOTTOM", - /* 179 */ "STDDEV", - /* 180 */ "PERCENTILE", - /* 181 */ "APERCENTILE", - /* 182 */ "LEASTSQUARES", - /* 183 */ "HISTOGRAM", - /* 184 */ "DIFF", - /* 185 */ "SPREAD", - /* 186 */ "TWA", - /* 187 */ "INTERP", - /* 188 */ "LAST_ROW", - /* 189 */ "RATE", - /* 190 */ "IRATE", - /* 191 */ "SUM_RATE", - /* 192 */ "SUM_IRATE", - /* 193 */ "AVG_RATE", - /* 194 */ "AVG_IRATE", - /* 195 */ "TBID", - /* 196 */ "SEMI", - /* 197 */ "NONE", - /* 198 */ "PREV", - /* 199 */ "LINEAR", - /* 200 */ "IMPORT", - /* 201 */ "METRIC", - /* 202 */ "TBNAME", - /* 203 */ "JOIN", - /* 204 */ "METRICS", - /* 205 */ "STABLE", - /* 206 */ "INSERT", - /* 207 */ "INTO", - /* 208 */ "VALUES", - /* 209 */ "error", - /* 210 */ "program", - /* 211 */ "cmd", - /* 212 */ "dbPrefix", - /* 213 */ "ids", - /* 214 */ "cpxName", - /* 215 */ "ifexists", - /* 216 */ "alter_db_optr", - /* 217 */ "acct_optr", - /* 218 */ "ifnotexists", - /* 219 */ "db_optr", - /* 220 */ "pps", - /* 221 */ "tseries", - /* 222 */ "dbs", - /* 223 */ "streams", - /* 224 */ "storage", - /* 225 */ "qtime", - /* 226 */ "users", - /* 227 */ "conns", - /* 228 */ "state", - /* 229 */ "keep", - /* 230 */ "tagitemlist", - /* 231 */ "cache", - /* 232 */ "replica", - /* 233 */ "quorum", - /* 234 */ "days", - /* 235 */ "minrows", - /* 236 */ "maxrows", - /* 237 */ "blocks", - /* 238 */ "ctime", - /* 239 */ "wal", - /* 240 */ "fsync", - /* 241 */ "comp", - /* 242 */ "prec", - /* 243 */ "update", - /* 244 */ "cachelast", - /* 245 */ "typename", - /* 246 */ "signed", - /* 247 */ "create_table_args", - /* 248 */ "create_table_list", - /* 249 */ "create_from_stable", - /* 250 */ "columnlist", - /* 251 */ "select", - /* 252 */ "column", - /* 253 */ "tagitem", - /* 254 */ "selcollist", - /* 255 */ "from", - /* 256 */ "where_opt", - /* 257 */ "interval_opt", - /* 258 */ "fill_opt", - /* 259 */ "sliding_opt", - /* 260 */ "groupby_opt", - /* 261 */ "orderby_opt", - /* 262 */ "having_opt", - /* 263 */ "slimit_opt", - /* 264 */ "limit_opt", - /* 265 */ "union", - /* 266 */ "sclp", - /* 267 */ "expr", - /* 268 */ "as", - /* 269 */ "tablelist", - /* 270 */ "tmvar", - /* 271 */ "sortlist", - /* 272 */ "sortitem", - /* 273 */ "item", - /* 274 */ "sortorder", - /* 275 */ "grouplist", - /* 276 */ "exprlist", - /* 277 */ "expritem", + /* 46 */ "TOPICS", + /* 47 */ "MNODES", + /* 48 */ "DNODES", + /* 49 */ "ACCOUNTS", + /* 50 */ "USERS", + /* 51 */ "MODULES", + /* 52 */ "QUERIES", + /* 53 */ "CONNECTIONS", + /* 54 */ "STREAMS", + /* 55 */ "VARIABLES", + /* 56 */ "SCORES", + /* 57 */ "GRANTS", + /* 58 */ "VNODES", + /* 59 */ "IPTOKEN", + /* 60 */ "DOT", + /* 61 */ "CREATE", + /* 62 */ "TABLE", + /* 63 */ "DATABASE", + /* 64 */ "TABLES", + /* 65 */ "STABLES", + /* 66 */ "VGROUPS", + /* 67 */ "DROP", + /* 68 */ "STABLE", + /* 69 */ "TOPIC", + /* 70 */ "DNODE", + /* 71 */ "USER", + /* 72 */ "ACCOUNT", + /* 73 */ "USE", + /* 74 */ "DESCRIBE", + /* 75 */ "ALTER", + /* 76 */ "PASS", + /* 77 */ "PRIVILEGE", + /* 78 */ "LOCAL", + /* 79 */ "IF", + /* 80 */ "EXISTS", + /* 81 */ "PPS", + /* 82 */ "TSERIES", + /* 83 */ "DBS", + /* 84 */ "STORAGE", + /* 85 */ "QTIME", + /* 86 */ "CONNS", + /* 87 */ "STATE", + /* 88 */ "KEEP", + /* 89 */ "CACHE", + /* 90 */ "REPLICA", + /* 91 */ "QUORUM", + /* 92 */ "DAYS", + /* 93 */ "MINROWS", + /* 94 */ "MAXROWS", + /* 95 */ "BLOCKS", + /* 96 */ "CTIME", + /* 97 */ "WAL", + /* 98 */ "FSYNC", + /* 99 */ "COMP", + /* 100 */ "PRECISION", + /* 101 */ "UPDATE", + /* 102 */ "CACHELAST", + /* 103 */ "PARTITIONS", + /* 104 */ "LP", + /* 105 */ "RP", + /* 106 */ "UNSIGNED", + /* 107 */ "TAGS", + /* 108 */ "USING", + /* 109 */ "COMMA", + /* 110 */ "AS", + /* 111 */ "NULL", + /* 112 */ "SELECT", + /* 113 */ "UNION", + /* 114 */ "ALL", + /* 115 */ "DISTINCT", + /* 116 */ "FROM", + /* 117 */ "VARIABLE", + /* 118 */ "INTERVAL", + /* 119 */ "SESSION", + /* 120 */ "FILL", + /* 121 */ "SLIDING", + /* 122 */ "ORDER", + /* 123 */ "BY", + /* 124 */ "ASC", + /* 125 */ "DESC", + /* 126 */ "GROUP", + /* 127 */ "HAVING", + /* 128 */ "LIMIT", + /* 129 */ "OFFSET", + /* 130 */ "SLIMIT", + /* 131 */ "SOFFSET", + /* 132 */ "WHERE", + /* 133 */ "NOW", + /* 134 */ "RESET", + /* 135 */ "QUERY", + /* 136 */ "SYNCDB", + /* 137 */ "ADD", + /* 138 */ "COLUMN", + /* 139 */ "TAG", + /* 140 */ "CHANGE", + /* 141 */ "SET", + /* 142 */ "KILL", + /* 143 */ "CONNECTION", + /* 144 */ "STREAM", + /* 145 */ "COLON", + /* 146 */ "ABORT", + /* 147 */ "AFTER", + /* 148 */ "ATTACH", + /* 149 */ "BEFORE", + /* 150 */ "BEGIN", + /* 151 */ "CASCADE", + /* 152 */ "CLUSTER", + /* 153 */ "CONFLICT", + /* 154 */ "COPY", + /* 155 */ "DEFERRED", + /* 156 */ "DELIMITERS", + /* 157 */ "DETACH", + /* 158 */ "EACH", + /* 159 */ "END", + /* 160 */ "EXPLAIN", + /* 161 */ "FAIL", + /* 162 */ "FOR", + /* 163 */ "IGNORE", + /* 164 */ "IMMEDIATE", + /* 165 */ "INITIALLY", + /* 166 */ "INSTEAD", + /* 167 */ "MATCH", + /* 168 */ "KEY", + /* 169 */ "OF", + /* 170 */ "RAISE", + /* 171 */ "REPLACE", + /* 172 */ "RESTRICT", + /* 173 */ "ROW", + /* 174 */ "STATEMENT", + /* 175 */ "TRIGGER", + /* 176 */ "VIEW", + /* 177 */ "SEMI", + /* 178 */ "NONE", + /* 179 */ "PREV", + /* 180 */ "LINEAR", + /* 181 */ "IMPORT", + /* 182 */ "TBNAME", + /* 183 */ "JOIN", + /* 184 */ "INSERT", + /* 185 */ "INTO", + /* 186 */ "VALUES", + /* 187 */ "error", + /* 188 */ "program", + /* 189 */ "cmd", + /* 190 */ "dbPrefix", + /* 191 */ "ids", + /* 192 */ "cpxName", + /* 193 */ "ifexists", + /* 194 */ "alter_db_optr", + /* 195 */ "alter_topic_optr", + /* 196 */ "acct_optr", + /* 197 */ "ifnotexists", + /* 198 */ "db_optr", + /* 199 */ "topic_optr", + /* 200 */ "pps", + /* 201 */ "tseries", + /* 202 */ "dbs", + /* 203 */ "streams", + /* 204 */ "storage", + /* 205 */ "qtime", + /* 206 */ "users", + /* 207 */ "conns", + /* 208 */ "state", + /* 209 */ "keep", + /* 210 */ "tagitemlist", + /* 211 */ "cache", + /* 212 */ "replica", + /* 213 */ "quorum", + /* 214 */ "days", + /* 215 */ "minrows", + /* 216 */ "maxrows", + /* 217 */ "blocks", + /* 218 */ "ctime", + /* 219 */ "wal", + /* 220 */ "fsync", + /* 221 */ "comp", + /* 222 */ "prec", + /* 223 */ "update", + /* 224 */ "cachelast", + /* 225 */ "partitions", + /* 226 */ "typename", + /* 227 */ "signed", + /* 228 */ "create_table_args", + /* 229 */ "create_stable_args", + /* 230 */ "create_table_list", + /* 231 */ "create_from_stable", + /* 232 */ "columnlist", + /* 233 */ "tagNamelist", + /* 234 */ "select", + /* 235 */ "column", + /* 236 */ "tagitem", + /* 237 */ "selcollist", + /* 238 */ "from", + /* 239 */ "where_opt", + /* 240 */ "interval_opt", + /* 241 */ "session_option", + /* 242 */ "fill_opt", + /* 243 */ "sliding_opt", + /* 244 */ "groupby_opt", + /* 245 */ "orderby_opt", + /* 246 */ "having_opt", + /* 247 */ "slimit_opt", + /* 248 */ "limit_opt", + /* 249 */ "union", + /* 250 */ "sclp", + /* 251 */ "distinct", + /* 252 */ "expr", + /* 253 */ "as", + /* 254 */ "tablelist", + /* 255 */ "tmvar", + /* 256 */ "sortlist", + /* 257 */ "sortitem", + /* 258 */ "item", + /* 259 */ "sortorder", + /* 260 */ "grouplist", + /* 261 */ "exprlist", + /* 262 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1026,244 +1021,271 @@ static const char *const yyTokenName[] = { static const char *const yyRuleName[] = { /* 0 */ "program ::= cmd", /* 1 */ "cmd ::= SHOW DATABASES", - /* 2 */ "cmd ::= SHOW MNODES", - /* 3 */ "cmd ::= SHOW DNODES", - /* 4 */ "cmd ::= SHOW ACCOUNTS", - /* 5 */ "cmd ::= SHOW USERS", - /* 6 */ "cmd ::= SHOW MODULES", - /* 7 */ "cmd ::= SHOW QUERIES", - /* 8 */ "cmd ::= SHOW CONNECTIONS", - /* 9 */ "cmd ::= SHOW STREAMS", - /* 10 */ "cmd ::= SHOW VARIABLES", - /* 11 */ "cmd ::= SHOW SCORES", - /* 12 */ "cmd ::= SHOW GRANTS", - /* 13 */ "cmd ::= SHOW VNODES", - /* 14 */ "cmd ::= SHOW VNODES IPTOKEN", - /* 15 */ "dbPrefix ::=", - /* 16 */ "dbPrefix ::= ids DOT", - /* 17 */ "cpxName ::=", - /* 18 */ "cpxName ::= DOT ids", - /* 19 */ "cmd ::= SHOW CREATE TABLE ids cpxName", - /* 20 */ "cmd ::= SHOW CREATE DATABASE ids", - /* 21 */ "cmd ::= SHOW dbPrefix TABLES", - /* 22 */ "cmd ::= SHOW dbPrefix TABLES LIKE ids", - /* 23 */ "cmd ::= SHOW dbPrefix STABLES", - /* 24 */ "cmd ::= SHOW dbPrefix STABLES LIKE ids", - /* 25 */ "cmd ::= SHOW dbPrefix VGROUPS", - /* 26 */ "cmd ::= SHOW dbPrefix VGROUPS ids", - /* 27 */ "cmd ::= DROP TABLE ifexists ids cpxName", - /* 28 */ "cmd ::= DROP DATABASE ifexists ids", - /* 29 */ "cmd ::= DROP DNODE ids", - /* 30 */ "cmd ::= DROP USER ids", - /* 31 */ "cmd ::= DROP ACCOUNT ids", - /* 32 */ "cmd ::= USE ids", - /* 33 */ "cmd ::= DESCRIBE ids cpxName", - /* 34 */ "cmd ::= ALTER USER ids PASS ids", - /* 35 */ "cmd ::= ALTER USER ids PRIVILEGE ids", - /* 36 */ "cmd ::= ALTER DNODE ids ids", - /* 37 */ "cmd ::= ALTER DNODE ids ids ids", - /* 38 */ "cmd ::= ALTER LOCAL ids", - /* 39 */ "cmd ::= ALTER LOCAL ids ids", - /* 40 */ "cmd ::= ALTER DATABASE ids alter_db_optr", - /* 41 */ "cmd ::= ALTER ACCOUNT ids acct_optr", - /* 42 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", - /* 43 */ "ids ::= ID", - /* 44 */ "ids ::= STRING", - /* 45 */ "ifexists ::= IF EXISTS", - /* 46 */ "ifexists ::=", - /* 47 */ "ifnotexists ::= IF NOT EXISTS", - /* 48 */ "ifnotexists ::=", - /* 49 */ "cmd ::= CREATE DNODE ids", - /* 50 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", - /* 51 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", - /* 52 */ "cmd ::= CREATE USER ids PASS ids", - /* 53 */ "pps ::=", - /* 54 */ "pps ::= PPS INTEGER", - /* 55 */ "tseries ::=", - /* 56 */ "tseries ::= TSERIES INTEGER", - /* 57 */ "dbs ::=", - /* 58 */ "dbs ::= DBS INTEGER", - /* 59 */ "streams ::=", - /* 60 */ "streams ::= STREAMS INTEGER", - /* 61 */ "storage ::=", - /* 62 */ "storage ::= STORAGE INTEGER", - /* 63 */ "qtime ::=", - /* 64 */ "qtime ::= QTIME INTEGER", - /* 65 */ "users ::=", - /* 66 */ "users ::= USERS INTEGER", - /* 67 */ "conns ::=", - /* 68 */ "conns ::= CONNS INTEGER", - /* 69 */ "state ::=", - /* 70 */ "state ::= STATE ids", - /* 71 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", - /* 72 */ "keep ::= KEEP tagitemlist", - /* 73 */ "cache ::= CACHE INTEGER", - /* 74 */ "replica ::= REPLICA INTEGER", - /* 75 */ "quorum ::= QUORUM INTEGER", - /* 76 */ "days ::= DAYS INTEGER", - /* 77 */ "minrows ::= MINROWS INTEGER", - /* 78 */ "maxrows ::= MAXROWS INTEGER", - /* 79 */ "blocks ::= BLOCKS INTEGER", - /* 80 */ "ctime ::= CTIME INTEGER", - /* 81 */ "wal ::= WAL INTEGER", - /* 82 */ "fsync ::= FSYNC INTEGER", - /* 83 */ "comp ::= COMP INTEGER", - /* 84 */ "prec ::= PRECISION STRING", - /* 85 */ "update ::= UPDATE INTEGER", - /* 86 */ "cachelast ::= CACHELAST INTEGER", - /* 87 */ "db_optr ::=", - /* 88 */ "db_optr ::= db_optr cache", - /* 89 */ "db_optr ::= db_optr replica", - /* 90 */ "db_optr ::= db_optr quorum", - /* 91 */ "db_optr ::= db_optr days", - /* 92 */ "db_optr ::= db_optr minrows", - /* 93 */ "db_optr ::= db_optr maxrows", - /* 94 */ "db_optr ::= db_optr blocks", - /* 95 */ "db_optr ::= db_optr ctime", - /* 96 */ "db_optr ::= db_optr wal", - /* 97 */ "db_optr ::= db_optr fsync", - /* 98 */ "db_optr ::= db_optr comp", - /* 99 */ "db_optr ::= db_optr prec", - /* 100 */ "db_optr ::= db_optr keep", - /* 101 */ "db_optr ::= db_optr update", - /* 102 */ "db_optr ::= db_optr cachelast", - /* 103 */ "alter_db_optr ::=", - /* 104 */ "alter_db_optr ::= alter_db_optr replica", - /* 105 */ "alter_db_optr ::= alter_db_optr quorum", - /* 106 */ "alter_db_optr ::= alter_db_optr keep", - /* 107 */ "alter_db_optr ::= alter_db_optr blocks", - /* 108 */ "alter_db_optr ::= alter_db_optr comp", - /* 109 */ "alter_db_optr ::= alter_db_optr wal", - /* 110 */ "alter_db_optr ::= alter_db_optr fsync", - /* 111 */ "alter_db_optr ::= alter_db_optr update", - /* 112 */ "alter_db_optr ::= alter_db_optr cachelast", - /* 113 */ "typename ::= ids", - /* 114 */ "typename ::= ids LP signed RP", - /* 115 */ "typename ::= ids UNSIGNED", - /* 116 */ "signed ::= INTEGER", - /* 117 */ "signed ::= PLUS INTEGER", - /* 118 */ "signed ::= MINUS INTEGER", - /* 119 */ "cmd ::= CREATE TABLE create_table_args", - /* 120 */ "cmd ::= CREATE TABLE create_table_list", - /* 121 */ "create_table_list ::= create_from_stable", - /* 122 */ "create_table_list ::= create_table_list create_from_stable", - /* 123 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", - /* 124 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", - /* 125 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", - /* 126 */ "create_table_args ::= ifnotexists ids cpxName AS select", - /* 127 */ "columnlist ::= columnlist COMMA column", - /* 128 */ "columnlist ::= column", - /* 129 */ "column ::= ids typename", - /* 130 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 131 */ "tagitemlist ::= tagitem", - /* 132 */ "tagitem ::= INTEGER", - /* 133 */ "tagitem ::= FLOAT", - /* 134 */ "tagitem ::= STRING", - /* 135 */ "tagitem ::= BOOL", - /* 136 */ "tagitem ::= NULL", - /* 137 */ "tagitem ::= MINUS INTEGER", - /* 138 */ "tagitem ::= MINUS FLOAT", - /* 139 */ "tagitem ::= PLUS INTEGER", - /* 140 */ "tagitem ::= PLUS FLOAT", - /* 141 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", - /* 142 */ "union ::= select", - /* 143 */ "union ::= LP union RP", - /* 144 */ "union ::= union UNION ALL select", - /* 145 */ "union ::= union UNION ALL LP select RP", - /* 146 */ "cmd ::= union", - /* 147 */ "select ::= SELECT selcollist", - /* 148 */ "sclp ::= selcollist COMMA", - /* 149 */ "sclp ::=", - /* 150 */ "selcollist ::= sclp expr as", - /* 151 */ "selcollist ::= sclp STAR", - /* 152 */ "as ::= AS ids", - /* 153 */ "as ::= ids", - /* 154 */ "as ::=", - /* 155 */ "from ::= FROM tablelist", - /* 156 */ "tablelist ::= ids cpxName", - /* 157 */ "tablelist ::= ids cpxName ids", - /* 158 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 159 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 160 */ "tmvar ::= VARIABLE", - /* 161 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 162 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", - /* 163 */ "interval_opt ::=", - /* 164 */ "fill_opt ::=", - /* 165 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 166 */ "fill_opt ::= FILL LP ID RP", - /* 167 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 168 */ "sliding_opt ::=", - /* 169 */ "orderby_opt ::=", - /* 170 */ "orderby_opt ::= ORDER BY sortlist", - /* 171 */ "sortlist ::= sortlist COMMA item sortorder", - /* 172 */ "sortlist ::= item sortorder", - /* 173 */ "item ::= ids cpxName", - /* 174 */ "sortorder ::= ASC", - /* 175 */ "sortorder ::= DESC", - /* 176 */ "sortorder ::=", - /* 177 */ "groupby_opt ::=", - /* 178 */ "groupby_opt ::= GROUP BY grouplist", - /* 179 */ "grouplist ::= grouplist COMMA item", - /* 180 */ "grouplist ::= item", - /* 181 */ "having_opt ::=", - /* 182 */ "having_opt ::= HAVING expr", - /* 183 */ "limit_opt ::=", - /* 184 */ "limit_opt ::= LIMIT signed", - /* 185 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 186 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 187 */ "slimit_opt ::=", - /* 188 */ "slimit_opt ::= SLIMIT signed", - /* 189 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 190 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 191 */ "where_opt ::=", - /* 192 */ "where_opt ::= WHERE expr", - /* 193 */ "expr ::= LP expr RP", - /* 194 */ "expr ::= ID", - /* 195 */ "expr ::= ID DOT ID", - /* 196 */ "expr ::= ID DOT STAR", - /* 197 */ "expr ::= INTEGER", - /* 198 */ "expr ::= MINUS INTEGER", - /* 199 */ "expr ::= PLUS INTEGER", - /* 200 */ "expr ::= FLOAT", - /* 201 */ "expr ::= MINUS FLOAT", - /* 202 */ "expr ::= PLUS FLOAT", - /* 203 */ "expr ::= STRING", - /* 204 */ "expr ::= NOW", - /* 205 */ "expr ::= VARIABLE", - /* 206 */ "expr ::= BOOL", - /* 207 */ "expr ::= ID LP exprlist RP", - /* 208 */ "expr ::= ID LP STAR RP", - /* 209 */ "expr ::= expr IS NULL", - /* 210 */ "expr ::= expr IS NOT NULL", - /* 211 */ "expr ::= expr LT expr", - /* 212 */ "expr ::= expr GT expr", - /* 213 */ "expr ::= expr LE expr", - /* 214 */ "expr ::= expr GE expr", - /* 215 */ "expr ::= expr NE expr", - /* 216 */ "expr ::= expr EQ expr", - /* 217 */ "expr ::= expr AND expr", - /* 218 */ "expr ::= expr OR expr", - /* 219 */ "expr ::= expr PLUS expr", - /* 220 */ "expr ::= expr MINUS expr", - /* 221 */ "expr ::= expr STAR expr", - /* 222 */ "expr ::= expr SLASH expr", - /* 223 */ "expr ::= expr REM expr", - /* 224 */ "expr ::= expr LIKE expr", - /* 225 */ "expr ::= expr IN LP exprlist RP", - /* 226 */ "exprlist ::= exprlist COMMA expritem", - /* 227 */ "exprlist ::= expritem", - /* 228 */ "expritem ::= expr", - /* 229 */ "expritem ::=", - /* 230 */ "cmd ::= RESET QUERY CACHE", - /* 231 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 232 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 233 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 234 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 235 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 236 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 237 */ "cmd ::= KILL CONNECTION INTEGER", - /* 238 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 239 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 2 */ "cmd ::= SHOW TOPICS", + /* 3 */ "cmd ::= SHOW MNODES", + /* 4 */ "cmd ::= SHOW DNODES", + /* 5 */ "cmd ::= SHOW ACCOUNTS", + /* 6 */ "cmd ::= SHOW USERS", + /* 7 */ "cmd ::= SHOW MODULES", + /* 8 */ "cmd ::= SHOW QUERIES", + /* 9 */ "cmd ::= SHOW CONNECTIONS", + /* 10 */ "cmd ::= SHOW STREAMS", + /* 11 */ "cmd ::= SHOW VARIABLES", + /* 12 */ "cmd ::= SHOW SCORES", + /* 13 */ "cmd ::= SHOW GRANTS", + /* 14 */ "cmd ::= SHOW VNODES", + /* 15 */ "cmd ::= SHOW VNODES IPTOKEN", + /* 16 */ "dbPrefix ::=", + /* 17 */ "dbPrefix ::= ids DOT", + /* 18 */ "cpxName ::=", + /* 19 */ "cpxName ::= DOT ids", + /* 20 */ "cmd ::= SHOW CREATE TABLE ids cpxName", + /* 21 */ "cmd ::= SHOW CREATE DATABASE ids", + /* 22 */ "cmd ::= SHOW dbPrefix TABLES", + /* 23 */ "cmd ::= SHOW dbPrefix TABLES LIKE ids", + /* 24 */ "cmd ::= SHOW dbPrefix STABLES", + /* 25 */ "cmd ::= SHOW dbPrefix STABLES LIKE ids", + /* 26 */ "cmd ::= SHOW dbPrefix VGROUPS", + /* 27 */ "cmd ::= SHOW dbPrefix VGROUPS ids", + /* 28 */ "cmd ::= DROP TABLE ifexists ids cpxName", + /* 29 */ "cmd ::= DROP STABLE ifexists ids cpxName", + /* 30 */ "cmd ::= DROP DATABASE ifexists ids", + /* 31 */ "cmd ::= DROP TOPIC ifexists ids", + /* 32 */ "cmd ::= DROP DNODE ids", + /* 33 */ "cmd ::= DROP USER ids", + /* 34 */ "cmd ::= DROP ACCOUNT ids", + /* 35 */ "cmd ::= USE ids", + /* 36 */ "cmd ::= DESCRIBE ids cpxName", + /* 37 */ "cmd ::= ALTER USER ids PASS ids", + /* 38 */ "cmd ::= ALTER USER ids PRIVILEGE ids", + /* 39 */ "cmd ::= ALTER DNODE ids ids", + /* 40 */ "cmd ::= ALTER DNODE ids ids ids", + /* 41 */ "cmd ::= ALTER LOCAL ids", + /* 42 */ "cmd ::= ALTER LOCAL ids ids", + /* 43 */ "cmd ::= ALTER DATABASE ids alter_db_optr", + /* 44 */ "cmd ::= ALTER TOPIC ids alter_topic_optr", + /* 45 */ "cmd ::= ALTER ACCOUNT ids acct_optr", + /* 46 */ "cmd ::= ALTER ACCOUNT ids PASS ids acct_optr", + /* 47 */ "ids ::= ID", + /* 48 */ "ids ::= STRING", + /* 49 */ "ifexists ::= IF EXISTS", + /* 50 */ "ifexists ::=", + /* 51 */ "ifnotexists ::= IF NOT EXISTS", + /* 52 */ "ifnotexists ::=", + /* 53 */ "cmd ::= CREATE DNODE ids", + /* 54 */ "cmd ::= CREATE ACCOUNT ids PASS ids acct_optr", + /* 55 */ "cmd ::= CREATE DATABASE ifnotexists ids db_optr", + /* 56 */ "cmd ::= CREATE TOPIC ifnotexists ids topic_optr", + /* 57 */ "cmd ::= CREATE USER ids PASS ids", + /* 58 */ "pps ::=", + /* 59 */ "pps ::= PPS INTEGER", + /* 60 */ "tseries ::=", + /* 61 */ "tseries ::= TSERIES INTEGER", + /* 62 */ "dbs ::=", + /* 63 */ "dbs ::= DBS INTEGER", + /* 64 */ "streams ::=", + /* 65 */ "streams ::= STREAMS INTEGER", + /* 66 */ "storage ::=", + /* 67 */ "storage ::= STORAGE INTEGER", + /* 68 */ "qtime ::=", + /* 69 */ "qtime ::= QTIME INTEGER", + /* 70 */ "users ::=", + /* 71 */ "users ::= USERS INTEGER", + /* 72 */ "conns ::=", + /* 73 */ "conns ::= CONNS INTEGER", + /* 74 */ "state ::=", + /* 75 */ "state ::= STATE ids", + /* 76 */ "acct_optr ::= pps tseries storage streams qtime dbs users conns state", + /* 77 */ "keep ::= KEEP tagitemlist", + /* 78 */ "cache ::= CACHE INTEGER", + /* 79 */ "replica ::= REPLICA INTEGER", + /* 80 */ "quorum ::= QUORUM INTEGER", + /* 81 */ "days ::= DAYS INTEGER", + /* 82 */ "minrows ::= MINROWS INTEGER", + /* 83 */ "maxrows ::= MAXROWS INTEGER", + /* 84 */ "blocks ::= BLOCKS INTEGER", + /* 85 */ "ctime ::= CTIME INTEGER", + /* 86 */ "wal ::= WAL INTEGER", + /* 87 */ "fsync ::= FSYNC INTEGER", + /* 88 */ "comp ::= COMP INTEGER", + /* 89 */ "prec ::= PRECISION STRING", + /* 90 */ "update ::= UPDATE INTEGER", + /* 91 */ "cachelast ::= CACHELAST INTEGER", + /* 92 */ "partitions ::= PARTITIONS INTEGER", + /* 93 */ "db_optr ::=", + /* 94 */ "db_optr ::= db_optr cache", + /* 95 */ "db_optr ::= db_optr replica", + /* 96 */ "db_optr ::= db_optr quorum", + /* 97 */ "db_optr ::= db_optr days", + /* 98 */ "db_optr ::= db_optr minrows", + /* 99 */ "db_optr ::= db_optr maxrows", + /* 100 */ "db_optr ::= db_optr blocks", + /* 101 */ "db_optr ::= db_optr ctime", + /* 102 */ "db_optr ::= db_optr wal", + /* 103 */ "db_optr ::= db_optr fsync", + /* 104 */ "db_optr ::= db_optr comp", + /* 105 */ "db_optr ::= db_optr prec", + /* 106 */ "db_optr ::= db_optr keep", + /* 107 */ "db_optr ::= db_optr update", + /* 108 */ "db_optr ::= db_optr cachelast", + /* 109 */ "topic_optr ::= db_optr", + /* 110 */ "topic_optr ::= topic_optr partitions", + /* 111 */ "alter_db_optr ::=", + /* 112 */ "alter_db_optr ::= alter_db_optr replica", + /* 113 */ "alter_db_optr ::= alter_db_optr quorum", + /* 114 */ "alter_db_optr ::= alter_db_optr keep", + /* 115 */ "alter_db_optr ::= alter_db_optr blocks", + /* 116 */ "alter_db_optr ::= alter_db_optr comp", + /* 117 */ "alter_db_optr ::= alter_db_optr wal", + /* 118 */ "alter_db_optr ::= alter_db_optr fsync", + /* 119 */ "alter_db_optr ::= alter_db_optr update", + /* 120 */ "alter_db_optr ::= alter_db_optr cachelast", + /* 121 */ "alter_topic_optr ::= alter_db_optr", + /* 122 */ "alter_topic_optr ::= alter_topic_optr partitions", + /* 123 */ "typename ::= ids", + /* 124 */ "typename ::= ids LP signed RP", + /* 125 */ "typename ::= ids UNSIGNED", + /* 126 */ "signed ::= INTEGER", + /* 127 */ "signed ::= PLUS INTEGER", + /* 128 */ "signed ::= MINUS INTEGER", + /* 129 */ "cmd ::= CREATE TABLE create_table_args", + /* 130 */ "cmd ::= CREATE TABLE create_stable_args", + /* 131 */ "cmd ::= CREATE STABLE create_stable_args", + /* 132 */ "cmd ::= CREATE TABLE create_table_list", + /* 133 */ "create_table_list ::= create_from_stable", + /* 134 */ "create_table_list ::= create_table_list create_from_stable", + /* 135 */ "create_table_args ::= ifnotexists ids cpxName LP columnlist RP", + /* 136 */ "create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP", + /* 137 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP", + /* 138 */ "create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP", + /* 139 */ "tagNamelist ::= tagNamelist COMMA ids", + /* 140 */ "tagNamelist ::= ids", + /* 141 */ "create_table_args ::= ifnotexists ids cpxName AS select", + /* 142 */ "columnlist ::= columnlist COMMA column", + /* 143 */ "columnlist ::= column", + /* 144 */ "column ::= ids typename", + /* 145 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 146 */ "tagitemlist ::= tagitem", + /* 147 */ "tagitem ::= INTEGER", + /* 148 */ "tagitem ::= FLOAT", + /* 149 */ "tagitem ::= STRING", + /* 150 */ "tagitem ::= BOOL", + /* 151 */ "tagitem ::= NULL", + /* 152 */ "tagitem ::= MINUS INTEGER", + /* 153 */ "tagitem ::= MINUS FLOAT", + /* 154 */ "tagitem ::= PLUS INTEGER", + /* 155 */ "tagitem ::= PLUS FLOAT", + /* 156 */ "select ::= SELECT selcollist from where_opt interval_opt session_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 157 */ "select ::= LP select RP", + /* 158 */ "union ::= select", + /* 159 */ "union ::= union UNION ALL select", + /* 160 */ "cmd ::= union", + /* 161 */ "select ::= SELECT selcollist", + /* 162 */ "sclp ::= selcollist COMMA", + /* 163 */ "sclp ::=", + /* 164 */ "selcollist ::= sclp distinct expr as", + /* 165 */ "selcollist ::= sclp STAR", + /* 166 */ "as ::= AS ids", + /* 167 */ "as ::= ids", + /* 168 */ "as ::=", + /* 169 */ "distinct ::= DISTINCT", + /* 170 */ "distinct ::=", + /* 171 */ "from ::= FROM tablelist", + /* 172 */ "from ::= FROM LP union RP", + /* 173 */ "tablelist ::= ids cpxName", + /* 174 */ "tablelist ::= ids cpxName ids", + /* 175 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 176 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 177 */ "tmvar ::= VARIABLE", + /* 178 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 179 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 180 */ "interval_opt ::=", + /* 181 */ "session_option ::=", + /* 182 */ "session_option ::= SESSION LP ids cpxName COMMA tmvar RP", + /* 183 */ "fill_opt ::=", + /* 184 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 185 */ "fill_opt ::= FILL LP ID RP", + /* 186 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 187 */ "sliding_opt ::=", + /* 188 */ "orderby_opt ::=", + /* 189 */ "orderby_opt ::= ORDER BY sortlist", + /* 190 */ "sortlist ::= sortlist COMMA item sortorder", + /* 191 */ "sortlist ::= item sortorder", + /* 192 */ "item ::= ids cpxName", + /* 193 */ "sortorder ::= ASC", + /* 194 */ "sortorder ::= DESC", + /* 195 */ "sortorder ::=", + /* 196 */ "groupby_opt ::=", + /* 197 */ "groupby_opt ::= GROUP BY grouplist", + /* 198 */ "grouplist ::= grouplist COMMA item", + /* 199 */ "grouplist ::= item", + /* 200 */ "having_opt ::=", + /* 201 */ "having_opt ::= HAVING expr", + /* 202 */ "limit_opt ::=", + /* 203 */ "limit_opt ::= LIMIT signed", + /* 204 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 205 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 206 */ "slimit_opt ::=", + /* 207 */ "slimit_opt ::= SLIMIT signed", + /* 208 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 209 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 210 */ "where_opt ::=", + /* 211 */ "where_opt ::= WHERE expr", + /* 212 */ "expr ::= LP expr RP", + /* 213 */ "expr ::= ID", + /* 214 */ "expr ::= ID DOT ID", + /* 215 */ "expr ::= ID DOT STAR", + /* 216 */ "expr ::= INTEGER", + /* 217 */ "expr ::= MINUS INTEGER", + /* 218 */ "expr ::= PLUS INTEGER", + /* 219 */ "expr ::= FLOAT", + /* 220 */ "expr ::= MINUS FLOAT", + /* 221 */ "expr ::= PLUS FLOAT", + /* 222 */ "expr ::= STRING", + /* 223 */ "expr ::= NOW", + /* 224 */ "expr ::= VARIABLE", + /* 225 */ "expr ::= BOOL", + /* 226 */ "expr ::= NULL", + /* 227 */ "expr ::= ID LP exprlist RP", + /* 228 */ "expr ::= ID LP STAR RP", + /* 229 */ "expr ::= expr IS NULL", + /* 230 */ "expr ::= expr IS NOT NULL", + /* 231 */ "expr ::= expr LT expr", + /* 232 */ "expr ::= expr GT expr", + /* 233 */ "expr ::= expr LE expr", + /* 234 */ "expr ::= expr GE expr", + /* 235 */ "expr ::= expr NE expr", + /* 236 */ "expr ::= expr EQ expr", + /* 237 */ "expr ::= expr BETWEEN expr AND expr", + /* 238 */ "expr ::= expr AND expr", + /* 239 */ "expr ::= expr OR expr", + /* 240 */ "expr ::= expr PLUS expr", + /* 241 */ "expr ::= expr MINUS expr", + /* 242 */ "expr ::= expr STAR expr", + /* 243 */ "expr ::= expr SLASH expr", + /* 244 */ "expr ::= expr REM expr", + /* 245 */ "expr ::= expr LIKE expr", + /* 246 */ "expr ::= expr IN LP exprlist RP", + /* 247 */ "exprlist ::= exprlist COMMA expritem", + /* 248 */ "exprlist ::= expritem", + /* 249 */ "expritem ::= expr", + /* 250 */ "expritem ::=", + /* 251 */ "cmd ::= RESET QUERY CACHE", + /* 252 */ "cmd ::= SYNCDB ids REPLICA", + /* 253 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 254 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 255 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 256 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 257 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 258 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 259 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", + /* 260 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", + /* 261 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", + /* 262 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", + /* 263 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", + /* 264 */ "cmd ::= KILL CONNECTION INTEGER", + /* 265 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 266 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1384,51 +1406,52 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 229: /* keep */ - case 230: /* tagitemlist */ - case 250: /* columnlist */ - case 258: /* fill_opt */ - case 260: /* groupby_opt */ - case 261: /* orderby_opt */ - case 271: /* sortlist */ - case 275: /* grouplist */ + case 209: /* keep */ + case 210: /* tagitemlist */ + case 232: /* columnlist */ + case 233: /* tagNamelist */ + case 242: /* fill_opt */ + case 244: /* groupby_opt */ + case 245: /* orderby_opt */ + case 256: /* sortlist */ + case 260: /* grouplist */ { -taosArrayDestroy((yypminor->yy421)); +taosArrayDestroy((yypminor->yy159)); } break; - case 248: /* create_table_list */ + case 230: /* create_table_list */ { -destroyCreateTableSql((yypminor->yy38)); +destroyCreateTableSql((yypminor->yy14)); } break; - case 251: /* select */ + case 234: /* select */ { -doDestroyQuerySql((yypminor->yy148)); +destroyQuerySqlNode((yypminor->yy272)); } break; - case 254: /* selcollist */ - case 266: /* sclp */ - case 276: /* exprlist */ + case 237: /* selcollist */ + case 250: /* sclp */ + case 261: /* exprlist */ { -tSqlExprListDestroy((yypminor->yy166)); +tSqlExprListDestroy((yypminor->yy159)); } break; - case 256: /* where_opt */ - case 262: /* having_opt */ - case 267: /* expr */ - case 277: /* expritem */ + case 239: /* where_opt */ + case 246: /* having_opt */ + case 252: /* expr */ + case 262: /* expritem */ { -tSqlExprDestroy((yypminor->yy78)); +tSqlExprDestroy((yypminor->yy118)); } break; - case 265: /* union */ + case 249: /* union */ { -destroyAllSelectClause((yypminor->yy153)); +destroyAllSelectClause((yypminor->yy391)); } break; - case 272: /* sortitem */ + case 257: /* sortitem */ { -tVariantDestroy(&(yypminor->yy430)); +tVariantDestroy(&(yypminor->yy488)); } break; /********* End destructor definitions *****************************************/ @@ -1722,246 +1745,273 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 210, -1 }, /* (0) program ::= cmd */ - { 211, -2 }, /* (1) cmd ::= SHOW DATABASES */ - { 211, -2 }, /* (2) cmd ::= SHOW MNODES */ - { 211, -2 }, /* (3) cmd ::= SHOW DNODES */ - { 211, -2 }, /* (4) cmd ::= SHOW ACCOUNTS */ - { 211, -2 }, /* (5) cmd ::= SHOW USERS */ - { 211, -2 }, /* (6) cmd ::= SHOW MODULES */ - { 211, -2 }, /* (7) cmd ::= SHOW QUERIES */ - { 211, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ - { 211, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 211, -2 }, /* (10) cmd ::= SHOW VARIABLES */ - { 211, -2 }, /* (11) cmd ::= SHOW SCORES */ - { 211, -2 }, /* (12) cmd ::= SHOW GRANTS */ - { 211, -2 }, /* (13) cmd ::= SHOW VNODES */ - { 211, -3 }, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - { 212, 0 }, /* (15) dbPrefix ::= */ - { 212, -2 }, /* (16) dbPrefix ::= ids DOT */ - { 214, 0 }, /* (17) cpxName ::= */ - { 214, -2 }, /* (18) cpxName ::= DOT ids */ - { 211, -5 }, /* (19) cmd ::= SHOW CREATE TABLE ids cpxName */ - { 211, -4 }, /* (20) cmd ::= SHOW CREATE DATABASE ids */ - { 211, -3 }, /* (21) cmd ::= SHOW dbPrefix TABLES */ - { 211, -5 }, /* (22) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - { 211, -3 }, /* (23) cmd ::= SHOW dbPrefix STABLES */ - { 211, -5 }, /* (24) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - { 211, -3 }, /* (25) cmd ::= SHOW dbPrefix VGROUPS */ - { 211, -4 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS ids */ - { 211, -5 }, /* (27) cmd ::= DROP TABLE ifexists ids cpxName */ - { 211, -4 }, /* (28) cmd ::= DROP DATABASE ifexists ids */ - { 211, -3 }, /* (29) cmd ::= DROP DNODE ids */ - { 211, -3 }, /* (30) cmd ::= DROP USER ids */ - { 211, -3 }, /* (31) cmd ::= DROP ACCOUNT ids */ - { 211, -2 }, /* (32) cmd ::= USE ids */ - { 211, -3 }, /* (33) cmd ::= DESCRIBE ids cpxName */ - { 211, -5 }, /* (34) cmd ::= ALTER USER ids PASS ids */ - { 211, -5 }, /* (35) cmd ::= ALTER USER ids PRIVILEGE ids */ - { 211, -4 }, /* (36) cmd ::= ALTER DNODE ids ids */ - { 211, -5 }, /* (37) cmd ::= ALTER DNODE ids ids ids */ - { 211, -3 }, /* (38) cmd ::= ALTER LOCAL ids */ - { 211, -4 }, /* (39) cmd ::= ALTER LOCAL ids ids */ - { 211, -4 }, /* (40) cmd ::= ALTER DATABASE ids alter_db_optr */ - { 211, -4 }, /* (41) cmd ::= ALTER ACCOUNT ids acct_optr */ - { 211, -6 }, /* (42) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - { 213, -1 }, /* (43) ids ::= ID */ - { 213, -1 }, /* (44) ids ::= STRING */ - { 215, -2 }, /* (45) ifexists ::= IF EXISTS */ - { 215, 0 }, /* (46) ifexists ::= */ - { 218, -3 }, /* (47) ifnotexists ::= IF NOT EXISTS */ - { 218, 0 }, /* (48) ifnotexists ::= */ - { 211, -3 }, /* (49) cmd ::= CREATE DNODE ids */ - { 211, -6 }, /* (50) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - { 211, -5 }, /* (51) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - { 211, -5 }, /* (52) cmd ::= CREATE USER ids PASS ids */ - { 220, 0 }, /* (53) pps ::= */ - { 220, -2 }, /* (54) pps ::= PPS INTEGER */ - { 221, 0 }, /* (55) tseries ::= */ - { 221, -2 }, /* (56) tseries ::= TSERIES INTEGER */ - { 222, 0 }, /* (57) dbs ::= */ - { 222, -2 }, /* (58) dbs ::= DBS INTEGER */ - { 223, 0 }, /* (59) streams ::= */ - { 223, -2 }, /* (60) streams ::= STREAMS INTEGER */ - { 224, 0 }, /* (61) storage ::= */ - { 224, -2 }, /* (62) storage ::= STORAGE INTEGER */ - { 225, 0 }, /* (63) qtime ::= */ - { 225, -2 }, /* (64) qtime ::= QTIME INTEGER */ - { 226, 0 }, /* (65) users ::= */ - { 226, -2 }, /* (66) users ::= USERS INTEGER */ - { 227, 0 }, /* (67) conns ::= */ - { 227, -2 }, /* (68) conns ::= CONNS INTEGER */ - { 228, 0 }, /* (69) state ::= */ - { 228, -2 }, /* (70) state ::= STATE ids */ - { 217, -9 }, /* (71) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - { 229, -2 }, /* (72) keep ::= KEEP tagitemlist */ - { 231, -2 }, /* (73) cache ::= CACHE INTEGER */ - { 232, -2 }, /* (74) replica ::= REPLICA INTEGER */ - { 233, -2 }, /* (75) quorum ::= QUORUM INTEGER */ - { 234, -2 }, /* (76) days ::= DAYS INTEGER */ - { 235, -2 }, /* (77) minrows ::= MINROWS INTEGER */ - { 236, -2 }, /* (78) maxrows ::= MAXROWS INTEGER */ - { 237, -2 }, /* (79) blocks ::= BLOCKS INTEGER */ - { 238, -2 }, /* (80) ctime ::= CTIME INTEGER */ - { 239, -2 }, /* (81) wal ::= WAL INTEGER */ - { 240, -2 }, /* (82) fsync ::= FSYNC INTEGER */ - { 241, -2 }, /* (83) comp ::= COMP INTEGER */ - { 242, -2 }, /* (84) prec ::= PRECISION STRING */ - { 243, -2 }, /* (85) update ::= UPDATE INTEGER */ - { 244, -2 }, /* (86) cachelast ::= CACHELAST INTEGER */ - { 219, 0 }, /* (87) db_optr ::= */ - { 219, -2 }, /* (88) db_optr ::= db_optr cache */ - { 219, -2 }, /* (89) db_optr ::= db_optr replica */ - { 219, -2 }, /* (90) db_optr ::= db_optr quorum */ - { 219, -2 }, /* (91) db_optr ::= db_optr days */ - { 219, -2 }, /* (92) db_optr ::= db_optr minrows */ - { 219, -2 }, /* (93) db_optr ::= db_optr maxrows */ - { 219, -2 }, /* (94) db_optr ::= db_optr blocks */ - { 219, -2 }, /* (95) db_optr ::= db_optr ctime */ - { 219, -2 }, /* (96) db_optr ::= db_optr wal */ - { 219, -2 }, /* (97) db_optr ::= db_optr fsync */ - { 219, -2 }, /* (98) db_optr ::= db_optr comp */ - { 219, -2 }, /* (99) db_optr ::= db_optr prec */ - { 219, -2 }, /* (100) db_optr ::= db_optr keep */ - { 219, -2 }, /* (101) db_optr ::= db_optr update */ - { 219, -2 }, /* (102) db_optr ::= db_optr cachelast */ - { 216, 0 }, /* (103) alter_db_optr ::= */ - { 216, -2 }, /* (104) alter_db_optr ::= alter_db_optr replica */ - { 216, -2 }, /* (105) alter_db_optr ::= alter_db_optr quorum */ - { 216, -2 }, /* (106) alter_db_optr ::= alter_db_optr keep */ - { 216, -2 }, /* (107) alter_db_optr ::= alter_db_optr blocks */ - { 216, -2 }, /* (108) alter_db_optr ::= alter_db_optr comp */ - { 216, -2 }, /* (109) alter_db_optr ::= alter_db_optr wal */ - { 216, -2 }, /* (110) alter_db_optr ::= alter_db_optr fsync */ - { 216, -2 }, /* (111) alter_db_optr ::= alter_db_optr update */ - { 216, -2 }, /* (112) alter_db_optr ::= alter_db_optr cachelast */ - { 245, -1 }, /* (113) typename ::= ids */ - { 245, -4 }, /* (114) typename ::= ids LP signed RP */ - { 245, -2 }, /* (115) typename ::= ids UNSIGNED */ - { 246, -1 }, /* (116) signed ::= INTEGER */ - { 246, -2 }, /* (117) signed ::= PLUS INTEGER */ - { 246, -2 }, /* (118) signed ::= MINUS INTEGER */ - { 211, -3 }, /* (119) cmd ::= CREATE TABLE create_table_args */ - { 211, -3 }, /* (120) cmd ::= CREATE TABLE create_table_list */ - { 248, -1 }, /* (121) create_table_list ::= create_from_stable */ - { 248, -2 }, /* (122) create_table_list ::= create_table_list create_from_stable */ - { 247, -6 }, /* (123) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - { 247, -10 }, /* (124) create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - { 249, -10 }, /* (125) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ - { 247, -5 }, /* (126) create_table_args ::= ifnotexists ids cpxName AS select */ - { 250, -3 }, /* (127) columnlist ::= columnlist COMMA column */ - { 250, -1 }, /* (128) columnlist ::= column */ - { 252, -2 }, /* (129) column ::= ids typename */ - { 230, -3 }, /* (130) tagitemlist ::= tagitemlist COMMA tagitem */ - { 230, -1 }, /* (131) tagitemlist ::= tagitem */ - { 253, -1 }, /* (132) tagitem ::= INTEGER */ - { 253, -1 }, /* (133) tagitem ::= FLOAT */ - { 253, -1 }, /* (134) tagitem ::= STRING */ - { 253, -1 }, /* (135) tagitem ::= BOOL */ - { 253, -1 }, /* (136) tagitem ::= NULL */ - { 253, -2 }, /* (137) tagitem ::= MINUS INTEGER */ - { 253, -2 }, /* (138) tagitem ::= MINUS FLOAT */ - { 253, -2 }, /* (139) tagitem ::= PLUS INTEGER */ - { 253, -2 }, /* (140) tagitem ::= PLUS FLOAT */ - { 251, -12 }, /* (141) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - { 265, -1 }, /* (142) union ::= select */ - { 265, -3 }, /* (143) union ::= LP union RP */ - { 265, -4 }, /* (144) union ::= union UNION ALL select */ - { 265, -6 }, /* (145) union ::= union UNION ALL LP select RP */ - { 211, -1 }, /* (146) cmd ::= union */ - { 251, -2 }, /* (147) select ::= SELECT selcollist */ - { 266, -2 }, /* (148) sclp ::= selcollist COMMA */ - { 266, 0 }, /* (149) sclp ::= */ - { 254, -3 }, /* (150) selcollist ::= sclp expr as */ - { 254, -2 }, /* (151) selcollist ::= sclp STAR */ - { 268, -2 }, /* (152) as ::= AS ids */ - { 268, -1 }, /* (153) as ::= ids */ - { 268, 0 }, /* (154) as ::= */ - { 255, -2 }, /* (155) from ::= FROM tablelist */ - { 269, -2 }, /* (156) tablelist ::= ids cpxName */ - { 269, -3 }, /* (157) tablelist ::= ids cpxName ids */ - { 269, -4 }, /* (158) tablelist ::= tablelist COMMA ids cpxName */ - { 269, -5 }, /* (159) tablelist ::= tablelist COMMA ids cpxName ids */ - { 270, -1 }, /* (160) tmvar ::= VARIABLE */ - { 257, -4 }, /* (161) interval_opt ::= INTERVAL LP tmvar RP */ - { 257, -6 }, /* (162) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - { 257, 0 }, /* (163) interval_opt ::= */ - { 258, 0 }, /* (164) fill_opt ::= */ - { 258, -6 }, /* (165) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 258, -4 }, /* (166) fill_opt ::= FILL LP ID RP */ - { 259, -4 }, /* (167) sliding_opt ::= SLIDING LP tmvar RP */ - { 259, 0 }, /* (168) sliding_opt ::= */ - { 261, 0 }, /* (169) orderby_opt ::= */ - { 261, -3 }, /* (170) orderby_opt ::= ORDER BY sortlist */ - { 271, -4 }, /* (171) sortlist ::= sortlist COMMA item sortorder */ - { 271, -2 }, /* (172) sortlist ::= item sortorder */ - { 273, -2 }, /* (173) item ::= ids cpxName */ - { 274, -1 }, /* (174) sortorder ::= ASC */ - { 274, -1 }, /* (175) sortorder ::= DESC */ - { 274, 0 }, /* (176) sortorder ::= */ - { 260, 0 }, /* (177) groupby_opt ::= */ - { 260, -3 }, /* (178) groupby_opt ::= GROUP BY grouplist */ - { 275, -3 }, /* (179) grouplist ::= grouplist COMMA item */ - { 275, -1 }, /* (180) grouplist ::= item */ - { 262, 0 }, /* (181) having_opt ::= */ - { 262, -2 }, /* (182) having_opt ::= HAVING expr */ - { 264, 0 }, /* (183) limit_opt ::= */ - { 264, -2 }, /* (184) limit_opt ::= LIMIT signed */ - { 264, -4 }, /* (185) limit_opt ::= LIMIT signed OFFSET signed */ - { 264, -4 }, /* (186) limit_opt ::= LIMIT signed COMMA signed */ - { 263, 0 }, /* (187) slimit_opt ::= */ - { 263, -2 }, /* (188) slimit_opt ::= SLIMIT signed */ - { 263, -4 }, /* (189) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 263, -4 }, /* (190) slimit_opt ::= SLIMIT signed COMMA signed */ - { 256, 0 }, /* (191) where_opt ::= */ - { 256, -2 }, /* (192) where_opt ::= WHERE expr */ - { 267, -3 }, /* (193) expr ::= LP expr RP */ - { 267, -1 }, /* (194) expr ::= ID */ - { 267, -3 }, /* (195) expr ::= ID DOT ID */ - { 267, -3 }, /* (196) expr ::= ID DOT STAR */ - { 267, -1 }, /* (197) expr ::= INTEGER */ - { 267, -2 }, /* (198) expr ::= MINUS INTEGER */ - { 267, -2 }, /* (199) expr ::= PLUS INTEGER */ - { 267, -1 }, /* (200) expr ::= FLOAT */ - { 267, -2 }, /* (201) expr ::= MINUS FLOAT */ - { 267, -2 }, /* (202) expr ::= PLUS FLOAT */ - { 267, -1 }, /* (203) expr ::= STRING */ - { 267, -1 }, /* (204) expr ::= NOW */ - { 267, -1 }, /* (205) expr ::= VARIABLE */ - { 267, -1 }, /* (206) expr ::= BOOL */ - { 267, -4 }, /* (207) expr ::= ID LP exprlist RP */ - { 267, -4 }, /* (208) expr ::= ID LP STAR RP */ - { 267, -3 }, /* (209) expr ::= expr IS NULL */ - { 267, -4 }, /* (210) expr ::= expr IS NOT NULL */ - { 267, -3 }, /* (211) expr ::= expr LT expr */ - { 267, -3 }, /* (212) expr ::= expr GT expr */ - { 267, -3 }, /* (213) expr ::= expr LE expr */ - { 267, -3 }, /* (214) expr ::= expr GE expr */ - { 267, -3 }, /* (215) expr ::= expr NE expr */ - { 267, -3 }, /* (216) expr ::= expr EQ expr */ - { 267, -3 }, /* (217) expr ::= expr AND expr */ - { 267, -3 }, /* (218) expr ::= expr OR expr */ - { 267, -3 }, /* (219) expr ::= expr PLUS expr */ - { 267, -3 }, /* (220) expr ::= expr MINUS expr */ - { 267, -3 }, /* (221) expr ::= expr STAR expr */ - { 267, -3 }, /* (222) expr ::= expr SLASH expr */ - { 267, -3 }, /* (223) expr ::= expr REM expr */ - { 267, -3 }, /* (224) expr ::= expr LIKE expr */ - { 267, -5 }, /* (225) expr ::= expr IN LP exprlist RP */ - { 276, -3 }, /* (226) exprlist ::= exprlist COMMA expritem */ - { 276, -1 }, /* (227) exprlist ::= expritem */ - { 277, -1 }, /* (228) expritem ::= expr */ - { 277, 0 }, /* (229) expritem ::= */ - { 211, -3 }, /* (230) cmd ::= RESET QUERY CACHE */ - { 211, -7 }, /* (231) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 211, -7 }, /* (232) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 211, -7 }, /* (233) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 211, -7 }, /* (234) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 211, -8 }, /* (235) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 211, -9 }, /* (236) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 211, -3 }, /* (237) cmd ::= KILL CONNECTION INTEGER */ - { 211, -5 }, /* (238) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 211, -5 }, /* (239) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 188, -1 }, /* (0) program ::= cmd */ + { 189, -2 }, /* (1) cmd ::= SHOW DATABASES */ + { 189, -2 }, /* (2) cmd ::= SHOW TOPICS */ + { 189, -2 }, /* (3) cmd ::= SHOW MNODES */ + { 189, -2 }, /* (4) cmd ::= SHOW DNODES */ + { 189, -2 }, /* (5) cmd ::= SHOW ACCOUNTS */ + { 189, -2 }, /* (6) cmd ::= SHOW USERS */ + { 189, -2 }, /* (7) cmd ::= SHOW MODULES */ + { 189, -2 }, /* (8) cmd ::= SHOW QUERIES */ + { 189, -2 }, /* (9) cmd ::= SHOW CONNECTIONS */ + { 189, -2 }, /* (10) cmd ::= SHOW STREAMS */ + { 189, -2 }, /* (11) cmd ::= SHOW VARIABLES */ + { 189, -2 }, /* (12) cmd ::= SHOW SCORES */ + { 189, -2 }, /* (13) cmd ::= SHOW GRANTS */ + { 189, -2 }, /* (14) cmd ::= SHOW VNODES */ + { 189, -3 }, /* (15) cmd ::= SHOW VNODES IPTOKEN */ + { 190, 0 }, /* (16) dbPrefix ::= */ + { 190, -2 }, /* (17) dbPrefix ::= ids DOT */ + { 192, 0 }, /* (18) cpxName ::= */ + { 192, -2 }, /* (19) cpxName ::= DOT ids */ + { 189, -5 }, /* (20) cmd ::= SHOW CREATE TABLE ids cpxName */ + { 189, -4 }, /* (21) cmd ::= SHOW CREATE DATABASE ids */ + { 189, -3 }, /* (22) cmd ::= SHOW dbPrefix TABLES */ + { 189, -5 }, /* (23) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + { 189, -3 }, /* (24) cmd ::= SHOW dbPrefix STABLES */ + { 189, -5 }, /* (25) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + { 189, -3 }, /* (26) cmd ::= SHOW dbPrefix VGROUPS */ + { 189, -4 }, /* (27) cmd ::= SHOW dbPrefix VGROUPS ids */ + { 189, -5 }, /* (28) cmd ::= DROP TABLE ifexists ids cpxName */ + { 189, -5 }, /* (29) cmd ::= DROP STABLE ifexists ids cpxName */ + { 189, -4 }, /* (30) cmd ::= DROP DATABASE ifexists ids */ + { 189, -4 }, /* (31) cmd ::= DROP TOPIC ifexists ids */ + { 189, -3 }, /* (32) cmd ::= DROP DNODE ids */ + { 189, -3 }, /* (33) cmd ::= DROP USER ids */ + { 189, -3 }, /* (34) cmd ::= DROP ACCOUNT ids */ + { 189, -2 }, /* (35) cmd ::= USE ids */ + { 189, -3 }, /* (36) cmd ::= DESCRIBE ids cpxName */ + { 189, -5 }, /* (37) cmd ::= ALTER USER ids PASS ids */ + { 189, -5 }, /* (38) cmd ::= ALTER USER ids PRIVILEGE ids */ + { 189, -4 }, /* (39) cmd ::= ALTER DNODE ids ids */ + { 189, -5 }, /* (40) cmd ::= ALTER DNODE ids ids ids */ + { 189, -3 }, /* (41) cmd ::= ALTER LOCAL ids */ + { 189, -4 }, /* (42) cmd ::= ALTER LOCAL ids ids */ + { 189, -4 }, /* (43) cmd ::= ALTER DATABASE ids alter_db_optr */ + { 189, -4 }, /* (44) cmd ::= ALTER TOPIC ids alter_topic_optr */ + { 189, -4 }, /* (45) cmd ::= ALTER ACCOUNT ids acct_optr */ + { 189, -6 }, /* (46) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + { 191, -1 }, /* (47) ids ::= ID */ + { 191, -1 }, /* (48) ids ::= STRING */ + { 193, -2 }, /* (49) ifexists ::= IF EXISTS */ + { 193, 0 }, /* (50) ifexists ::= */ + { 197, -3 }, /* (51) ifnotexists ::= IF NOT EXISTS */ + { 197, 0 }, /* (52) ifnotexists ::= */ + { 189, -3 }, /* (53) cmd ::= CREATE DNODE ids */ + { 189, -6 }, /* (54) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + { 189, -5 }, /* (55) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + { 189, -5 }, /* (56) cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ + { 189, -5 }, /* (57) cmd ::= CREATE USER ids PASS ids */ + { 200, 0 }, /* (58) pps ::= */ + { 200, -2 }, /* (59) pps ::= PPS INTEGER */ + { 201, 0 }, /* (60) tseries ::= */ + { 201, -2 }, /* (61) tseries ::= TSERIES INTEGER */ + { 202, 0 }, /* (62) dbs ::= */ + { 202, -2 }, /* (63) dbs ::= DBS INTEGER */ + { 203, 0 }, /* (64) streams ::= */ + { 203, -2 }, /* (65) streams ::= STREAMS INTEGER */ + { 204, 0 }, /* (66) storage ::= */ + { 204, -2 }, /* (67) storage ::= STORAGE INTEGER */ + { 205, 0 }, /* (68) qtime ::= */ + { 205, -2 }, /* (69) qtime ::= QTIME INTEGER */ + { 206, 0 }, /* (70) users ::= */ + { 206, -2 }, /* (71) users ::= USERS INTEGER */ + { 207, 0 }, /* (72) conns ::= */ + { 207, -2 }, /* (73) conns ::= CONNS INTEGER */ + { 208, 0 }, /* (74) state ::= */ + { 208, -2 }, /* (75) state ::= STATE ids */ + { 196, -9 }, /* (76) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + { 209, -2 }, /* (77) keep ::= KEEP tagitemlist */ + { 211, -2 }, /* (78) cache ::= CACHE INTEGER */ + { 212, -2 }, /* (79) replica ::= REPLICA INTEGER */ + { 213, -2 }, /* (80) quorum ::= QUORUM INTEGER */ + { 214, -2 }, /* (81) days ::= DAYS INTEGER */ + { 215, -2 }, /* (82) minrows ::= MINROWS INTEGER */ + { 216, -2 }, /* (83) maxrows ::= MAXROWS INTEGER */ + { 217, -2 }, /* (84) blocks ::= BLOCKS INTEGER */ + { 218, -2 }, /* (85) ctime ::= CTIME INTEGER */ + { 219, -2 }, /* (86) wal ::= WAL INTEGER */ + { 220, -2 }, /* (87) fsync ::= FSYNC INTEGER */ + { 221, -2 }, /* (88) comp ::= COMP INTEGER */ + { 222, -2 }, /* (89) prec ::= PRECISION STRING */ + { 223, -2 }, /* (90) update ::= UPDATE INTEGER */ + { 224, -2 }, /* (91) cachelast ::= CACHELAST INTEGER */ + { 225, -2 }, /* (92) partitions ::= PARTITIONS INTEGER */ + { 198, 0 }, /* (93) db_optr ::= */ + { 198, -2 }, /* (94) db_optr ::= db_optr cache */ + { 198, -2 }, /* (95) db_optr ::= db_optr replica */ + { 198, -2 }, /* (96) db_optr ::= db_optr quorum */ + { 198, -2 }, /* (97) db_optr ::= db_optr days */ + { 198, -2 }, /* (98) db_optr ::= db_optr minrows */ + { 198, -2 }, /* (99) db_optr ::= db_optr maxrows */ + { 198, -2 }, /* (100) db_optr ::= db_optr blocks */ + { 198, -2 }, /* (101) db_optr ::= db_optr ctime */ + { 198, -2 }, /* (102) db_optr ::= db_optr wal */ + { 198, -2 }, /* (103) db_optr ::= db_optr fsync */ + { 198, -2 }, /* (104) db_optr ::= db_optr comp */ + { 198, -2 }, /* (105) db_optr ::= db_optr prec */ + { 198, -2 }, /* (106) db_optr ::= db_optr keep */ + { 198, -2 }, /* (107) db_optr ::= db_optr update */ + { 198, -2 }, /* (108) db_optr ::= db_optr cachelast */ + { 199, -1 }, /* (109) topic_optr ::= db_optr */ + { 199, -2 }, /* (110) topic_optr ::= topic_optr partitions */ + { 194, 0 }, /* (111) alter_db_optr ::= */ + { 194, -2 }, /* (112) alter_db_optr ::= alter_db_optr replica */ + { 194, -2 }, /* (113) alter_db_optr ::= alter_db_optr quorum */ + { 194, -2 }, /* (114) alter_db_optr ::= alter_db_optr keep */ + { 194, -2 }, /* (115) alter_db_optr ::= alter_db_optr blocks */ + { 194, -2 }, /* (116) alter_db_optr ::= alter_db_optr comp */ + { 194, -2 }, /* (117) alter_db_optr ::= alter_db_optr wal */ + { 194, -2 }, /* (118) alter_db_optr ::= alter_db_optr fsync */ + { 194, -2 }, /* (119) alter_db_optr ::= alter_db_optr update */ + { 194, -2 }, /* (120) alter_db_optr ::= alter_db_optr cachelast */ + { 195, -1 }, /* (121) alter_topic_optr ::= alter_db_optr */ + { 195, -2 }, /* (122) alter_topic_optr ::= alter_topic_optr partitions */ + { 226, -1 }, /* (123) typename ::= ids */ + { 226, -4 }, /* (124) typename ::= ids LP signed RP */ + { 226, -2 }, /* (125) typename ::= ids UNSIGNED */ + { 227, -1 }, /* (126) signed ::= INTEGER */ + { 227, -2 }, /* (127) signed ::= PLUS INTEGER */ + { 227, -2 }, /* (128) signed ::= MINUS INTEGER */ + { 189, -3 }, /* (129) cmd ::= CREATE TABLE create_table_args */ + { 189, -3 }, /* (130) cmd ::= CREATE TABLE create_stable_args */ + { 189, -3 }, /* (131) cmd ::= CREATE STABLE create_stable_args */ + { 189, -3 }, /* (132) cmd ::= CREATE TABLE create_table_list */ + { 230, -1 }, /* (133) create_table_list ::= create_from_stable */ + { 230, -2 }, /* (134) create_table_list ::= create_table_list create_from_stable */ + { 228, -6 }, /* (135) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + { 229, -10 }, /* (136) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + { 231, -10 }, /* (137) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + { 231, -13 }, /* (138) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ + { 233, -3 }, /* (139) tagNamelist ::= tagNamelist COMMA ids */ + { 233, -1 }, /* (140) tagNamelist ::= ids */ + { 228, -5 }, /* (141) create_table_args ::= ifnotexists ids cpxName AS select */ + { 232, -3 }, /* (142) columnlist ::= columnlist COMMA column */ + { 232, -1 }, /* (143) columnlist ::= column */ + { 235, -2 }, /* (144) column ::= ids typename */ + { 210, -3 }, /* (145) tagitemlist ::= tagitemlist COMMA tagitem */ + { 210, -1 }, /* (146) tagitemlist ::= tagitem */ + { 236, -1 }, /* (147) tagitem ::= INTEGER */ + { 236, -1 }, /* (148) tagitem ::= FLOAT */ + { 236, -1 }, /* (149) tagitem ::= STRING */ + { 236, -1 }, /* (150) tagitem ::= BOOL */ + { 236, -1 }, /* (151) tagitem ::= NULL */ + { 236, -2 }, /* (152) tagitem ::= MINUS INTEGER */ + { 236, -2 }, /* (153) tagitem ::= MINUS FLOAT */ + { 236, -2 }, /* (154) tagitem ::= PLUS INTEGER */ + { 236, -2 }, /* (155) tagitem ::= PLUS FLOAT */ + { 234, -13 }, /* (156) select ::= SELECT selcollist from where_opt interval_opt session_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + { 234, -3 }, /* (157) select ::= LP select RP */ + { 249, -1 }, /* (158) union ::= select */ + { 249, -4 }, /* (159) union ::= union UNION ALL select */ + { 189, -1 }, /* (160) cmd ::= union */ + { 234, -2 }, /* (161) select ::= SELECT selcollist */ + { 250, -2 }, /* (162) sclp ::= selcollist COMMA */ + { 250, 0 }, /* (163) sclp ::= */ + { 237, -4 }, /* (164) selcollist ::= sclp distinct expr as */ + { 237, -2 }, /* (165) selcollist ::= sclp STAR */ + { 253, -2 }, /* (166) as ::= AS ids */ + { 253, -1 }, /* (167) as ::= ids */ + { 253, 0 }, /* (168) as ::= */ + { 251, -1 }, /* (169) distinct ::= DISTINCT */ + { 251, 0 }, /* (170) distinct ::= */ + { 238, -2 }, /* (171) from ::= FROM tablelist */ + { 238, -4 }, /* (172) from ::= FROM LP union RP */ + { 254, -2 }, /* (173) tablelist ::= ids cpxName */ + { 254, -3 }, /* (174) tablelist ::= ids cpxName ids */ + { 254, -4 }, /* (175) tablelist ::= tablelist COMMA ids cpxName */ + { 254, -5 }, /* (176) tablelist ::= tablelist COMMA ids cpxName ids */ + { 255, -1 }, /* (177) tmvar ::= VARIABLE */ + { 240, -4 }, /* (178) interval_opt ::= INTERVAL LP tmvar RP */ + { 240, -6 }, /* (179) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ + { 240, 0 }, /* (180) interval_opt ::= */ + { 241, 0 }, /* (181) session_option ::= */ + { 241, -7 }, /* (182) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + { 242, 0 }, /* (183) fill_opt ::= */ + { 242, -6 }, /* (184) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 242, -4 }, /* (185) fill_opt ::= FILL LP ID RP */ + { 243, -4 }, /* (186) sliding_opt ::= SLIDING LP tmvar RP */ + { 243, 0 }, /* (187) sliding_opt ::= */ + { 245, 0 }, /* (188) orderby_opt ::= */ + { 245, -3 }, /* (189) orderby_opt ::= ORDER BY sortlist */ + { 256, -4 }, /* (190) sortlist ::= sortlist COMMA item sortorder */ + { 256, -2 }, /* (191) sortlist ::= item sortorder */ + { 258, -2 }, /* (192) item ::= ids cpxName */ + { 259, -1 }, /* (193) sortorder ::= ASC */ + { 259, -1 }, /* (194) sortorder ::= DESC */ + { 259, 0 }, /* (195) sortorder ::= */ + { 244, 0 }, /* (196) groupby_opt ::= */ + { 244, -3 }, /* (197) groupby_opt ::= GROUP BY grouplist */ + { 260, -3 }, /* (198) grouplist ::= grouplist COMMA item */ + { 260, -1 }, /* (199) grouplist ::= item */ + { 246, 0 }, /* (200) having_opt ::= */ + { 246, -2 }, /* (201) having_opt ::= HAVING expr */ + { 248, 0 }, /* (202) limit_opt ::= */ + { 248, -2 }, /* (203) limit_opt ::= LIMIT signed */ + { 248, -4 }, /* (204) limit_opt ::= LIMIT signed OFFSET signed */ + { 248, -4 }, /* (205) limit_opt ::= LIMIT signed COMMA signed */ + { 247, 0 }, /* (206) slimit_opt ::= */ + { 247, -2 }, /* (207) slimit_opt ::= SLIMIT signed */ + { 247, -4 }, /* (208) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 247, -4 }, /* (209) slimit_opt ::= SLIMIT signed COMMA signed */ + { 239, 0 }, /* (210) where_opt ::= */ + { 239, -2 }, /* (211) where_opt ::= WHERE expr */ + { 252, -3 }, /* (212) expr ::= LP expr RP */ + { 252, -1 }, /* (213) expr ::= ID */ + { 252, -3 }, /* (214) expr ::= ID DOT ID */ + { 252, -3 }, /* (215) expr ::= ID DOT STAR */ + { 252, -1 }, /* (216) expr ::= INTEGER */ + { 252, -2 }, /* (217) expr ::= MINUS INTEGER */ + { 252, -2 }, /* (218) expr ::= PLUS INTEGER */ + { 252, -1 }, /* (219) expr ::= FLOAT */ + { 252, -2 }, /* (220) expr ::= MINUS FLOAT */ + { 252, -2 }, /* (221) expr ::= PLUS FLOAT */ + { 252, -1 }, /* (222) expr ::= STRING */ + { 252, -1 }, /* (223) expr ::= NOW */ + { 252, -1 }, /* (224) expr ::= VARIABLE */ + { 252, -1 }, /* (225) expr ::= BOOL */ + { 252, -1 }, /* (226) expr ::= NULL */ + { 252, -4 }, /* (227) expr ::= ID LP exprlist RP */ + { 252, -4 }, /* (228) expr ::= ID LP STAR RP */ + { 252, -3 }, /* (229) expr ::= expr IS NULL */ + { 252, -4 }, /* (230) expr ::= expr IS NOT NULL */ + { 252, -3 }, /* (231) expr ::= expr LT expr */ + { 252, -3 }, /* (232) expr ::= expr GT expr */ + { 252, -3 }, /* (233) expr ::= expr LE expr */ + { 252, -3 }, /* (234) expr ::= expr GE expr */ + { 252, -3 }, /* (235) expr ::= expr NE expr */ + { 252, -3 }, /* (236) expr ::= expr EQ expr */ + { 252, -5 }, /* (237) expr ::= expr BETWEEN expr AND expr */ + { 252, -3 }, /* (238) expr ::= expr AND expr */ + { 252, -3 }, /* (239) expr ::= expr OR expr */ + { 252, -3 }, /* (240) expr ::= expr PLUS expr */ + { 252, -3 }, /* (241) expr ::= expr MINUS expr */ + { 252, -3 }, /* (242) expr ::= expr STAR expr */ + { 252, -3 }, /* (243) expr ::= expr SLASH expr */ + { 252, -3 }, /* (244) expr ::= expr REM expr */ + { 252, -3 }, /* (245) expr ::= expr LIKE expr */ + { 252, -5 }, /* (246) expr ::= expr IN LP exprlist RP */ + { 261, -3 }, /* (247) exprlist ::= exprlist COMMA expritem */ + { 261, -1 }, /* (248) exprlist ::= expritem */ + { 262, -1 }, /* (249) expritem ::= expr */ + { 262, 0 }, /* (250) expritem ::= */ + { 189, -3 }, /* (251) cmd ::= RESET QUERY CACHE */ + { 189, -3 }, /* (252) cmd ::= SYNCDB ids REPLICA */ + { 189, -7 }, /* (253) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 189, -7 }, /* (254) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 189, -7 }, /* (255) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 189, -7 }, /* (256) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 189, -8 }, /* (257) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 189, -9 }, /* (258) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 189, -7 }, /* (259) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + { 189, -7 }, /* (260) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + { 189, -7 }, /* (261) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + { 189, -7 }, /* (262) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + { 189, -8 }, /* (263) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + { 189, -3 }, /* (264) cmd ::= KILL CONNECTION INTEGER */ + { 189, -5 }, /* (265) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 189, -5 }, /* (266) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2042,863 +2092,926 @@ static void yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* program ::= cmd */ - case 119: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==119); + case 129: /* cmd ::= CREATE TABLE create_table_args */ yytestcase(yyruleno==129); + case 130: /* cmd ::= CREATE TABLE create_stable_args */ yytestcase(yyruleno==130); + case 131: /* cmd ::= CREATE STABLE create_stable_args */ yytestcase(yyruleno==131); {} break; case 1: /* cmd ::= SHOW DATABASES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} break; - case 2: /* cmd ::= SHOW MNODES */ + case 2: /* cmd ::= SHOW TOPICS */ +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} + break; + case 3: /* cmd ::= SHOW MNODES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} break; - case 3: /* cmd ::= SHOW DNODES */ + case 4: /* cmd ::= SHOW DNODES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} break; - case 4: /* cmd ::= SHOW ACCOUNTS */ + case 5: /* cmd ::= SHOW ACCOUNTS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} break; - case 5: /* cmd ::= SHOW USERS */ + case 6: /* cmd ::= SHOW USERS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_USER, 0, 0);} break; - case 6: /* cmd ::= SHOW MODULES */ + case 7: /* cmd ::= SHOW MODULES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); } break; - case 7: /* cmd ::= SHOW QUERIES */ + case 8: /* cmd ::= SHOW QUERIES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } break; - case 8: /* cmd ::= SHOW CONNECTIONS */ + case 9: /* cmd ::= SHOW CONNECTIONS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} break; - case 9: /* cmd ::= SHOW STREAMS */ + case 10: /* cmd ::= SHOW STREAMS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } break; - case 10: /* cmd ::= SHOW VARIABLES */ + case 11: /* cmd ::= SHOW VARIABLES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } break; - case 11: /* cmd ::= SHOW SCORES */ + case 12: /* cmd ::= SHOW SCORES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } break; - case 12: /* cmd ::= SHOW GRANTS */ + case 13: /* cmd ::= SHOW GRANTS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } break; - case 13: /* cmd ::= SHOW VNODES */ + case 14: /* cmd ::= SHOW VNODES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } break; - case 14: /* cmd ::= SHOW VNODES IPTOKEN */ + case 15: /* cmd ::= SHOW VNODES IPTOKEN */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &yymsp[0].minor.yy0, 0); } break; - case 15: /* dbPrefix ::= */ + case 16: /* dbPrefix ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.type = 0;} break; - case 16: /* dbPrefix ::= ids DOT */ + case 17: /* dbPrefix ::= ids DOT */ {yylhsminor.yy0 = yymsp[-1].minor.yy0; } yymsp[-1].minor.yy0 = yylhsminor.yy0; break; - case 17: /* cpxName ::= */ + case 18: /* cpxName ::= */ {yymsp[1].minor.yy0.n = 0; } break; - case 18: /* cpxName ::= DOT ids */ + case 19: /* cpxName ::= DOT ids */ {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n += 1; } break; - case 19: /* cmd ::= SHOW CREATE TABLE ids cpxName */ + case 20: /* cmd ::= SHOW CREATE TABLE ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDCLSQLElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &yymsp[-1].minor.yy0); + setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &yymsp[-1].minor.yy0); } break; - case 20: /* cmd ::= SHOW CREATE DATABASE ids */ + case 21: /* cmd ::= SHOW CREATE DATABASE ids */ { - setDCLSQLElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &yymsp[0].minor.yy0); + setDCLSqlElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &yymsp[0].minor.yy0); } break; - case 21: /* cmd ::= SHOW dbPrefix TABLES */ + case 22: /* cmd ::= SHOW dbPrefix TABLES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &yymsp[-1].minor.yy0, 0); } break; - case 22: /* cmd ::= SHOW dbPrefix TABLES LIKE ids */ + case 23: /* cmd ::= SHOW dbPrefix TABLES LIKE ids */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0); } break; - case 23: /* cmd ::= SHOW dbPrefix STABLES */ + case 24: /* cmd ::= SHOW dbPrefix STABLES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &yymsp[-1].minor.yy0, 0); } break; - case 24: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ + case 25: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { SStrToken token; - setDbName(&token, &yymsp[-3].minor.yy0); + tSetDbName(&token, &yymsp[-3].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); } break; - case 25: /* cmd ::= SHOW dbPrefix VGROUPS */ + case 26: /* cmd ::= SHOW dbPrefix VGROUPS */ { SStrToken token; - setDbName(&token, &yymsp[-1].minor.yy0); + tSetDbName(&token, &yymsp[-1].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; - case 26: /* cmd ::= SHOW dbPrefix VGROUPS ids */ + case 27: /* cmd ::= SHOW dbPrefix VGROUPS ids */ { SStrToken token; - setDbName(&token, &yymsp[-2].minor.yy0); + tSetDbName(&token, &yymsp[-2].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); } break; - case 27: /* cmd ::= DROP TABLE ifexists ids cpxName */ + case 28: /* cmd ::= DROP TABLE ifexists ids cpxName */ +{ + yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1, -1); +} + break; + case 29: /* cmd ::= DROP STABLE ifexists ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0); + setDropDbTableInfo(pInfo, TSDB_SQL_DROP_TABLE, &yymsp[-1].minor.yy0, &yymsp[-2].minor.yy0, -1, TSDB_SUPER_TABLE); } break; - case 28: /* cmd ::= DROP DATABASE ifexists ids */ -{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0); } + case 30: /* cmd ::= DROP DATABASE ifexists ids */ +{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, TSDB_DB_TYPE_DEFAULT, -1); } break; - case 29: /* cmd ::= DROP DNODE ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &yymsp[0].minor.yy0); } + case 31: /* cmd ::= DROP TOPIC ifexists ids */ +{ setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &yymsp[0].minor.yy0, &yymsp[-1].minor.yy0, TSDB_DB_TYPE_TOPIC, -1); } break; - case 30: /* cmd ::= DROP USER ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_DROP_USER, 1, &yymsp[0].minor.yy0); } + case 32: /* cmd ::= DROP DNODE ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &yymsp[0].minor.yy0); } break; - case 31: /* cmd ::= DROP ACCOUNT ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &yymsp[0].minor.yy0); } + case 33: /* cmd ::= DROP USER ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &yymsp[0].minor.yy0); } break; - case 32: /* cmd ::= USE ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_USE_DB, 1, &yymsp[0].minor.yy0);} + case 34: /* cmd ::= DROP ACCOUNT ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_DROP_ACCT, 1, &yymsp[0].minor.yy0); } break; - case 33: /* cmd ::= DESCRIBE ids cpxName */ + case 35: /* cmd ::= USE ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_USE_DB, 1, &yymsp[0].minor.yy0);} + break; + case 36: /* cmd ::= DESCRIBE ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - setDCLSQLElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &yymsp[-1].minor.yy0); + setDCLSqlElems(pInfo, TSDB_SQL_DESCRIBE_TABLE, 1, &yymsp[-1].minor.yy0); } break; - case 34: /* cmd ::= ALTER USER ids PASS ids */ + case 37: /* cmd ::= ALTER USER ids PASS ids */ { setAlterUserSql(pInfo, TSDB_ALTER_USER_PASSWD, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; - case 35: /* cmd ::= ALTER USER ids PRIVILEGE ids */ + case 38: /* cmd ::= ALTER USER ids PRIVILEGE ids */ { setAlterUserSql(pInfo, TSDB_ALTER_USER_PRIVILEGES, &yymsp[-2].minor.yy0, NULL, &yymsp[0].minor.yy0);} break; - case 36: /* cmd ::= ALTER DNODE ids ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + case 39: /* cmd ::= ALTER DNODE ids ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 37: /* cmd ::= ALTER DNODE ids ids ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + case 40: /* cmd ::= ALTER DNODE ids ids ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 38: /* cmd ::= ALTER LOCAL ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &yymsp[0].minor.yy0); } + case 41: /* cmd ::= ALTER LOCAL ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &yymsp[0].minor.yy0); } break; - case 39: /* cmd ::= ALTER LOCAL ids ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } + case 42: /* cmd ::= ALTER LOCAL ids ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 40: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy234, &t);} + case 43: /* cmd ::= ALTER DATABASE ids alter_db_optr */ + case 44: /* cmd ::= ALTER TOPIC ids alter_topic_optr */ yytestcase(yyruleno==44); +{ SStrToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy322, &t);} break; - case 41: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy71);} + case 45: /* cmd ::= ALTER ACCOUNT ids acct_optr */ +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy351);} break; - case 42: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy71);} + case 46: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy351);} break; - case 43: /* ids ::= ID */ - case 44: /* ids ::= STRING */ yytestcase(yyruleno==44); + case 47: /* ids ::= ID */ + case 48: /* ids ::= STRING */ yytestcase(yyruleno==48); {yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 45: /* ifexists ::= IF EXISTS */ + case 49: /* ifexists ::= IF EXISTS */ { yymsp[-1].minor.yy0.n = 1;} break; - case 46: /* ifexists ::= */ - case 48: /* ifnotexists ::= */ yytestcase(yyruleno==48); + case 50: /* ifexists ::= */ + case 52: /* ifnotexists ::= */ yytestcase(yyruleno==52); + case 170: /* distinct ::= */ yytestcase(yyruleno==170); { yymsp[1].minor.yy0.n = 0;} break; - case 47: /* ifnotexists ::= IF NOT EXISTS */ + case 51: /* ifnotexists ::= IF NOT EXISTS */ { yymsp[-2].minor.yy0.n = 1;} break; - case 49: /* cmd ::= CREATE DNODE ids */ -{ setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} + case 53: /* cmd ::= CREATE DNODE ids */ +{ setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; - case 50: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy71);} + case 54: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ +{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy351);} break; - case 51: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy234, &yymsp[-2].minor.yy0);} + case 55: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + case 56: /* cmd ::= CREATE TOPIC ifnotexists ids topic_optr */ yytestcase(yyruleno==56); +{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy322, &yymsp[-2].minor.yy0);} break; - case 52: /* cmd ::= CREATE USER ids PASS ids */ + case 57: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} break; - case 53: /* pps ::= */ - case 55: /* tseries ::= */ yytestcase(yyruleno==55); - case 57: /* dbs ::= */ yytestcase(yyruleno==57); - case 59: /* streams ::= */ yytestcase(yyruleno==59); - case 61: /* storage ::= */ yytestcase(yyruleno==61); - case 63: /* qtime ::= */ yytestcase(yyruleno==63); - case 65: /* users ::= */ yytestcase(yyruleno==65); - case 67: /* conns ::= */ yytestcase(yyruleno==67); - case 69: /* state ::= */ yytestcase(yyruleno==69); + case 58: /* pps ::= */ + case 60: /* tseries ::= */ yytestcase(yyruleno==60); + case 62: /* dbs ::= */ yytestcase(yyruleno==62); + case 64: /* streams ::= */ yytestcase(yyruleno==64); + case 66: /* storage ::= */ yytestcase(yyruleno==66); + case 68: /* qtime ::= */ yytestcase(yyruleno==68); + case 70: /* users ::= */ yytestcase(yyruleno==70); + case 72: /* conns ::= */ yytestcase(yyruleno==72); + case 74: /* state ::= */ yytestcase(yyruleno==74); { yymsp[1].minor.yy0.n = 0; } break; - case 54: /* pps ::= PPS INTEGER */ - case 56: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==56); - case 58: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==58); - case 60: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==60); - case 62: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==62); - case 64: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==64); - case 66: /* users ::= USERS INTEGER */ yytestcase(yyruleno==66); - case 68: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==68); - case 70: /* state ::= STATE ids */ yytestcase(yyruleno==70); + case 59: /* pps ::= PPS INTEGER */ + case 61: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==61); + case 63: /* dbs ::= DBS INTEGER */ yytestcase(yyruleno==63); + case 65: /* streams ::= STREAMS INTEGER */ yytestcase(yyruleno==65); + case 67: /* storage ::= STORAGE INTEGER */ yytestcase(yyruleno==67); + case 69: /* qtime ::= QTIME INTEGER */ yytestcase(yyruleno==69); + case 71: /* users ::= USERS INTEGER */ yytestcase(yyruleno==71); + case 73: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==73); + case 75: /* state ::= STATE ids */ yytestcase(yyruleno==75); { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 71: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + case 76: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy71.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy71.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy71.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy71.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy71.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy71.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy71.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy71.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy71.stat = yymsp[0].minor.yy0; -} - yymsp[-8].minor.yy71 = yylhsminor.yy71; - break; - case 72: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy421 = yymsp[0].minor.yy421; } - break; - case 73: /* cache ::= CACHE INTEGER */ - case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74); - case 75: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==75); - case 76: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==76); - case 77: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==77); - case 78: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==78); - case 79: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==79); - case 80: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==80); - case 81: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==81); - case 82: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==82); - case 83: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==83); - case 84: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==84); - case 85: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==85); - case 86: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==86); + yylhsminor.yy351.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy351.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy351.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy351.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy351.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy351.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy351.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy351.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy351.stat = yymsp[0].minor.yy0; +} + yymsp[-8].minor.yy351 = yylhsminor.yy351; + break; + case 77: /* keep ::= KEEP tagitemlist */ +{ yymsp[-1].minor.yy159 = yymsp[0].minor.yy159; } + break; + case 78: /* cache ::= CACHE INTEGER */ + case 79: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==79); + case 80: /* quorum ::= QUORUM INTEGER */ yytestcase(yyruleno==80); + case 81: /* days ::= DAYS INTEGER */ yytestcase(yyruleno==81); + case 82: /* minrows ::= MINROWS INTEGER */ yytestcase(yyruleno==82); + case 83: /* maxrows ::= MAXROWS INTEGER */ yytestcase(yyruleno==83); + case 84: /* blocks ::= BLOCKS INTEGER */ yytestcase(yyruleno==84); + case 85: /* ctime ::= CTIME INTEGER */ yytestcase(yyruleno==85); + case 86: /* wal ::= WAL INTEGER */ yytestcase(yyruleno==86); + case 87: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==87); + case 88: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==88); + case 89: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==89); + case 90: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==90); + case 91: /* cachelast ::= CACHELAST INTEGER */ yytestcase(yyruleno==91); + case 92: /* partitions ::= PARTITIONS INTEGER */ yytestcase(yyruleno==92); { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 87: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy234);} - break; - case 88: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 89: /* db_optr ::= db_optr replica */ - case 104: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==104); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 90: /* db_optr ::= db_optr quorum */ - case 105: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==105); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 91: /* db_optr ::= db_optr days */ -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 92: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 93: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 94: /* db_optr ::= db_optr blocks */ - case 107: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==107); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 95: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 96: /* db_optr ::= db_optr wal */ - case 109: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==109); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 97: /* db_optr ::= db_optr fsync */ - case 110: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==110); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 98: /* db_optr ::= db_optr comp */ - case 108: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==108); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 99: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 100: /* db_optr ::= db_optr keep */ - case 106: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==106); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.keep = yymsp[0].minor.yy421; } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 101: /* db_optr ::= db_optr update */ - case 111: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==111); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 102: /* db_optr ::= db_optr cachelast */ - case 112: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==112); -{ yylhsminor.yy234 = yymsp[-1].minor.yy234; yylhsminor.yy234.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy234 = yylhsminor.yy234; - break; - case 103: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy234);} - break; - case 113: /* typename ::= ids */ + case 93: /* db_optr ::= */ +{setDefaultCreateDbOption(&yymsp[1].minor.yy322); yymsp[1].minor.yy322.dbType = TSDB_DB_TYPE_DEFAULT;} + break; + case 94: /* db_optr ::= db_optr cache */ +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 95: /* db_optr ::= db_optr replica */ + case 112: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==112); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 96: /* db_optr ::= db_optr quorum */ + case 113: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==113); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 97: /* db_optr ::= db_optr days */ +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 98: /* db_optr ::= db_optr minrows */ +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 99: /* db_optr ::= db_optr maxrows */ +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 100: /* db_optr ::= db_optr blocks */ + case 115: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==115); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 101: /* db_optr ::= db_optr ctime */ +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 102: /* db_optr ::= db_optr wal */ + case 117: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==117); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 103: /* db_optr ::= db_optr fsync */ + case 118: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==118); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 104: /* db_optr ::= db_optr comp */ + case 116: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==116); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 105: /* db_optr ::= db_optr prec */ +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 106: /* db_optr ::= db_optr keep */ + case 114: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==114); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.keep = yymsp[0].minor.yy159; } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 107: /* db_optr ::= db_optr update */ + case 119: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==119); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 108: /* db_optr ::= db_optr cachelast */ + case 120: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==120); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 109: /* topic_optr ::= db_optr */ + case 121: /* alter_topic_optr ::= alter_db_optr */ yytestcase(yyruleno==121); +{ yylhsminor.yy322 = yymsp[0].minor.yy322; yylhsminor.yy322.dbType = TSDB_DB_TYPE_TOPIC; } + yymsp[0].minor.yy322 = yylhsminor.yy322; + break; + case 110: /* topic_optr ::= topic_optr partitions */ + case 122: /* alter_topic_optr ::= alter_topic_optr partitions */ yytestcase(yyruleno==122); +{ yylhsminor.yy322 = yymsp[-1].minor.yy322; yylhsminor.yy322.partitions = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy322 = yylhsminor.yy322; + break; + case 111: /* alter_db_optr ::= */ +{ setDefaultCreateDbOption(&yymsp[1].minor.yy322); yymsp[1].minor.yy322.dbType = TSDB_DB_TYPE_DEFAULT;} + break; + case 123: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSqlSetColumnType (&yylhsminor.yy183, &yymsp[0].minor.yy0); + tSetColumnType (&yylhsminor.yy407, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy183 = yylhsminor.yy183; + yymsp[0].minor.yy407 = yylhsminor.yy407; break; - case 114: /* typename ::= ids LP signed RP */ + case 124: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy325 <= 0) { + if (yymsp[-1].minor.yy317 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSqlSetColumnType(&yylhsminor.yy183, &yymsp[-3].minor.yy0); + tSetColumnType(&yylhsminor.yy407, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy325; // negative value of name length - tSqlSetColumnType(&yylhsminor.yy183, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy317; // negative value of name length + tSetColumnType(&yylhsminor.yy407, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy183 = yylhsminor.yy183; + yymsp[-3].minor.yy407 = yylhsminor.yy407; break; - case 115: /* typename ::= ids UNSIGNED */ + case 125: /* typename ::= ids UNSIGNED */ { yymsp[-1].minor.yy0.type = 0; yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z); - tSqlSetColumnType (&yylhsminor.yy183, &yymsp[-1].minor.yy0); + tSetColumnType (&yylhsminor.yy407, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy183 = yylhsminor.yy183; + yymsp[-1].minor.yy407 = yylhsminor.yy407; break; - case 116: /* signed ::= INTEGER */ -{ yylhsminor.yy325 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy325 = yylhsminor.yy325; + case 126: /* signed ::= INTEGER */ +{ yylhsminor.yy317 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy317 = yylhsminor.yy317; break; - case 117: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy325 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 127: /* signed ::= PLUS INTEGER */ +{ yymsp[-1].minor.yy317 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 118: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy325 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} + case 128: /* signed ::= MINUS INTEGER */ +{ yymsp[-1].minor.yy317 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 120: /* cmd ::= CREATE TABLE create_table_list */ -{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy38;} + case 132: /* cmd ::= CREATE TABLE create_table_list */ +{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy14;} break; - case 121: /* create_table_list ::= create_from_stable */ + case 133: /* create_table_list ::= create_from_stable */ { - SCreateTableSQL* pCreateTable = calloc(1, sizeof(SCreateTableSQL)); + SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy152); + taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy206); pCreateTable->type = TSQL_CREATE_TABLE_FROM_STABLE; - yylhsminor.yy38 = pCreateTable; + yylhsminor.yy14 = pCreateTable; } - yymsp[0].minor.yy38 = yylhsminor.yy38; + yymsp[0].minor.yy14 = yylhsminor.yy14; break; - case 122: /* create_table_list ::= create_table_list create_from_stable */ + case 134: /* create_table_list ::= create_table_list create_from_stable */ { - taosArrayPush(yymsp[-1].minor.yy38->childTableInfo, &yymsp[0].minor.yy152); - yylhsminor.yy38 = yymsp[-1].minor.yy38; + taosArrayPush(yymsp[-1].minor.yy14->childTableInfo, &yymsp[0].minor.yy206); + yylhsminor.yy14 = yymsp[-1].minor.yy14; } - yymsp[-1].minor.yy38 = yylhsminor.yy38; + yymsp[-1].minor.yy14 = yylhsminor.yy14; break; - case 123: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + case 135: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ { - yylhsminor.yy38 = tSetCreateSqlElems(yymsp[-1].minor.yy421, NULL, NULL, TSQL_CREATE_TABLE); - setSqlInfo(pInfo, yylhsminor.yy38, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy14 = tSetCreateTableInfo(yymsp[-1].minor.yy159, NULL, NULL, TSQL_CREATE_TABLE); + setSqlInfo(pInfo, yylhsminor.yy14, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy38 = yylhsminor.yy38; + yymsp[-5].minor.yy14 = yylhsminor.yy14; break; - case 124: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + case 136: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ { - yylhsminor.yy38 = tSetCreateSqlElems(yymsp[-5].minor.yy421, yymsp[-1].minor.yy421, NULL, TSQL_CREATE_STABLE); - setSqlInfo(pInfo, yylhsminor.yy38, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy14 = tSetCreateTableInfo(yymsp[-5].minor.yy159, yymsp[-1].minor.yy159, NULL, TSQL_CREATE_STABLE); + setSqlInfo(pInfo, yylhsminor.yy14, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy38 = yylhsminor.yy38; + yymsp[-9].minor.yy14 = yylhsminor.yy14; break; - case 125: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ + case 137: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yylhsminor.yy152 = createNewChildTableInfo(&yymsp[-5].minor.yy0, yymsp[-1].minor.yy421, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); + yylhsminor.yy206 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy159, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy152 = yylhsminor.yy152; + yymsp[-9].minor.yy206 = yylhsminor.yy206; break; - case 126: /* create_table_args ::= ifnotexists ids cpxName AS select */ + case 138: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist RP */ { - yylhsminor.yy38 = tSetCreateSqlElems(NULL, NULL, yymsp[0].minor.yy148, TSQL_CREATE_STREAM); - setSqlInfo(pInfo, yylhsminor.yy38, NULL, TSDB_SQL_CREATE_TABLE); + yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; + yymsp[-11].minor.yy0.n += yymsp[-10].minor.yy0.n; + yylhsminor.yy206 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy159, yymsp[-1].minor.yy159, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); +} + yymsp[-12].minor.yy206 = yylhsminor.yy206; + break; + case 139: /* tagNamelist ::= tagNamelist COMMA ids */ +{taosArrayPush(yymsp[-2].minor.yy159, &yymsp[0].minor.yy0); yylhsminor.yy159 = yymsp[-2].minor.yy159; } + yymsp[-2].minor.yy159 = yylhsminor.yy159; + break; + case 140: /* tagNamelist ::= ids */ +{yylhsminor.yy159 = taosArrayInit(4, sizeof(SStrToken)); taosArrayPush(yylhsminor.yy159, &yymsp[0].minor.yy0);} + yymsp[0].minor.yy159 = yylhsminor.yy159; + break; + case 141: /* create_table_args ::= ifnotexists ids cpxName AS select */ +{ + yylhsminor.yy14 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy272, TSQL_CREATE_STREAM); + setSqlInfo(pInfo, yylhsminor.yy14, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy38 = yylhsminor.yy38; + yymsp[-4].minor.yy14 = yylhsminor.yy14; break; - case 127: /* columnlist ::= columnlist COMMA column */ -{taosArrayPush(yymsp[-2].minor.yy421, &yymsp[0].minor.yy183); yylhsminor.yy421 = yymsp[-2].minor.yy421; } - yymsp[-2].minor.yy421 = yylhsminor.yy421; + case 142: /* columnlist ::= columnlist COMMA column */ +{taosArrayPush(yymsp[-2].minor.yy159, &yymsp[0].minor.yy407); yylhsminor.yy159 = yymsp[-2].minor.yy159; } + yymsp[-2].minor.yy159 = yylhsminor.yy159; break; - case 128: /* columnlist ::= column */ -{yylhsminor.yy421 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy421, &yymsp[0].minor.yy183);} - yymsp[0].minor.yy421 = yylhsminor.yy421; + case 143: /* columnlist ::= column */ +{yylhsminor.yy159 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy159, &yymsp[0].minor.yy407);} + yymsp[0].minor.yy159 = yylhsminor.yy159; break; - case 129: /* column ::= ids typename */ + case 144: /* column ::= ids typename */ { - tSqlSetColumnInfo(&yylhsminor.yy183, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy183); + tSetColumnInfo(&yylhsminor.yy407, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy407); } - yymsp[-1].minor.yy183 = yylhsminor.yy183; + yymsp[-1].minor.yy407 = yylhsminor.yy407; break; - case 130: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy421 = tVariantListAppend(yymsp[-2].minor.yy421, &yymsp[0].minor.yy430, -1); } - yymsp[-2].minor.yy421 = yylhsminor.yy421; + case 145: /* tagitemlist ::= tagitemlist COMMA tagitem */ +{ yylhsminor.yy159 = tVariantListAppend(yymsp[-2].minor.yy159, &yymsp[0].minor.yy488, -1); } + yymsp[-2].minor.yy159 = yylhsminor.yy159; break; - case 131: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[0].minor.yy430, -1); } - yymsp[0].minor.yy421 = yylhsminor.yy421; + case 146: /* tagitemlist ::= tagitem */ +{ yylhsminor.yy159 = tVariantListAppend(NULL, &yymsp[0].minor.yy488, -1); } + yymsp[0].minor.yy159 = yylhsminor.yy159; break; - case 132: /* tagitem ::= INTEGER */ - case 133: /* tagitem ::= FLOAT */ yytestcase(yyruleno==133); - case 134: /* tagitem ::= STRING */ yytestcase(yyruleno==134); - case 135: /* tagitem ::= BOOL */ yytestcase(yyruleno==135); -{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy430 = yylhsminor.yy430; + case 147: /* tagitem ::= INTEGER */ + case 148: /* tagitem ::= FLOAT */ yytestcase(yyruleno==148); + case 149: /* tagitem ::= STRING */ yytestcase(yyruleno==149); + case 150: /* tagitem ::= BOOL */ yytestcase(yyruleno==150); +{ toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy488, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy488 = yylhsminor.yy488; break; - case 136: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy430, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy430 = yylhsminor.yy430; + case 151: /* tagitem ::= NULL */ +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy488, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy488 = yylhsminor.yy488; break; - case 137: /* tagitem ::= MINUS INTEGER */ - case 138: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==138); - case 139: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==139); - case 140: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==140); + case 152: /* tagitem ::= MINUS INTEGER */ + case 153: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==153); + case 154: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==154); + case 155: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==155); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy430, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy488, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy430 = yylhsminor.yy430; + yymsp[-1].minor.yy488 = yylhsminor.yy488; break; - case 141: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 156: /* select ::= SELECT selcollist from where_opt interval_opt session_option fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy148 = tSetQuerySqlElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy166, yymsp[-9].minor.yy421, yymsp[-8].minor.yy78, yymsp[-4].minor.yy421, yymsp[-3].minor.yy421, &yymsp[-7].minor.yy400, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy421, &yymsp[0].minor.yy167, &yymsp[-1].minor.yy167); + yylhsminor.yy272 = tSetQuerySqlNode(&yymsp[-12].minor.yy0, yymsp[-11].minor.yy159, yymsp[-10].minor.yy514, yymsp[-9].minor.yy118, yymsp[-4].minor.yy159, yymsp[-3].minor.yy159, &yymsp[-8].minor.yy184, &yymsp[-7].minor.yy249, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy159, &yymsp[0].minor.yy440, &yymsp[-1].minor.yy440); } - yymsp[-11].minor.yy148 = yylhsminor.yy148; + yymsp[-12].minor.yy272 = yylhsminor.yy272; break; - case 142: /* union ::= select */ -{ yylhsminor.yy153 = setSubclause(NULL, yymsp[0].minor.yy148); } - yymsp[0].minor.yy153 = yylhsminor.yy153; + case 157: /* select ::= LP select RP */ +{yymsp[-2].minor.yy272 = yymsp[-1].minor.yy272;} break; - case 143: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy153 = yymsp[-1].minor.yy153; } + case 158: /* union ::= select */ +{ yylhsminor.yy391 = setSubclause(NULL, yymsp[0].minor.yy272); } + yymsp[0].minor.yy391 = yylhsminor.yy391; break; - case 144: /* union ::= union UNION ALL select */ -{ yylhsminor.yy153 = appendSelectClause(yymsp[-3].minor.yy153, yymsp[0].minor.yy148); } - yymsp[-3].minor.yy153 = yylhsminor.yy153; + case 159: /* union ::= union UNION ALL select */ +{ yylhsminor.yy391 = appendSelectClause(yymsp[-3].minor.yy391, yymsp[0].minor.yy272); } + yymsp[-3].minor.yy391 = yylhsminor.yy391; break; - case 145: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy153 = appendSelectClause(yymsp[-5].minor.yy153, yymsp[-1].minor.yy148); } - yymsp[-5].minor.yy153 = yylhsminor.yy153; + case 160: /* cmd ::= union */ +{ setSqlInfo(pInfo, yymsp[0].minor.yy391, NULL, TSDB_SQL_SELECT); } break; - case 146: /* cmd ::= union */ -{ setSqlInfo(pInfo, yymsp[0].minor.yy153, NULL, TSDB_SQL_SELECT); } - break; - case 147: /* select ::= SELECT selcollist */ + case 161: /* select ::= SELECT selcollist */ { - yylhsminor.yy148 = tSetQuerySqlElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy166, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yylhsminor.yy272 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy159, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy148 = yylhsminor.yy148; + yymsp[-1].minor.yy272 = yylhsminor.yy272; break; - case 148: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy166 = yymsp[-1].minor.yy166;} - yymsp[-1].minor.yy166 = yylhsminor.yy166; + case 162: /* sclp ::= selcollist COMMA */ +{yylhsminor.yy159 = yymsp[-1].minor.yy159;} + yymsp[-1].minor.yy159 = yylhsminor.yy159; break; - case 149: /* sclp ::= */ -{yymsp[1].minor.yy166 = 0;} + case 163: /* sclp ::= */ + case 188: /* orderby_opt ::= */ yytestcase(yyruleno==188); +{yymsp[1].minor.yy159 = 0;} break; - case 150: /* selcollist ::= sclp expr as */ + case 164: /* selcollist ::= sclp distinct expr as */ { - yylhsminor.yy166 = tSqlExprListAppend(yymsp[-2].minor.yy166, yymsp[-1].minor.yy78, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yylhsminor.yy159 = tSqlExprListAppend(yymsp[-3].minor.yy159, yymsp[-1].minor.yy118, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-2].minor.yy166 = yylhsminor.yy166; + yymsp[-3].minor.yy159 = yylhsminor.yy159; break; - case 151: /* selcollist ::= sclp STAR */ + case 165: /* selcollist ::= sclp STAR */ { - tSQLExpr *pNode = tSqlExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy166 = tSqlExprListAppend(yymsp[-1].minor.yy166, pNode, 0); + tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); + yylhsminor.yy159 = tSqlExprListAppend(yymsp[-1].minor.yy159, pNode, 0, 0); } - yymsp[-1].minor.yy166 = yylhsminor.yy166; + yymsp[-1].minor.yy159 = yylhsminor.yy159; break; - case 152: /* as ::= AS ids */ + case 166: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } break; - case 153: /* as ::= ids */ + case 167: /* as ::= ids */ { yylhsminor.yy0 = yymsp[0].minor.yy0; } yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 154: /* as ::= */ + case 168: /* as ::= */ { yymsp[1].minor.yy0.n = 0; } break; - case 155: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy421 = yymsp[0].minor.yy421;} + case 169: /* distinct ::= DISTINCT */ +{ yylhsminor.yy0 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 156: /* tablelist ::= ids cpxName */ + case 171: /* from ::= FROM tablelist */ +{yymsp[-1].minor.yy514 = yymsp[0].minor.yy159;} + break; + case 172: /* from ::= FROM LP union RP */ +{yymsp[-3].minor.yy514 = yymsp[-1].minor.yy391;} + break; + case 173: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy421 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy421 = tVariantListAppendToken(yylhsminor.yy421, &yymsp[-1].minor.yy0, -1); // table alias name + yylhsminor.yy159 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); } - yymsp[-1].minor.yy421 = yylhsminor.yy421; + yymsp[-1].minor.yy159 = yylhsminor.yy159; break; - case 157: /* tablelist ::= ids cpxName ids */ + case 174: /* tablelist ::= ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy421 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy421 = tVariantListAppendToken(yylhsminor.yy421, &yymsp[0].minor.yy0, -1); + yylhsminor.yy159 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy421 = yylhsminor.yy421; + yymsp[-2].minor.yy159 = yylhsminor.yy159; break; - case 158: /* tablelist ::= tablelist COMMA ids cpxName */ + case 175: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy421 = tVariantListAppendToken(yymsp[-3].minor.yy421, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy421 = tVariantListAppendToken(yylhsminor.yy421, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy159 = setTableNameList(yymsp[-3].minor.yy159, &yymsp[-1].minor.yy0, NULL); } - yymsp[-3].minor.yy421 = yylhsminor.yy421; + yymsp[-3].minor.yy159 = yylhsminor.yy159; break; - case 159: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 176: /* tablelist ::= tablelist COMMA ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy421 = tVariantListAppendToken(yymsp[-4].minor.yy421, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy421 = tVariantListAppendToken(yylhsminor.yy421, &yymsp[0].minor.yy0, -1); + + yylhsminor.yy159 = setTableNameList(yymsp[-4].minor.yy159, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } - yymsp[-4].minor.yy421 = yylhsminor.yy421; + yymsp[-4].minor.yy159 = yylhsminor.yy159; break; - case 160: /* tmvar ::= VARIABLE */ + case 177: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 161: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy400.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy400.offset.n = 0; yymsp[-3].minor.yy400.offset.z = NULL; yymsp[-3].minor.yy400.offset.type = 0;} + case 178: /* interval_opt ::= INTERVAL LP tmvar RP */ +{yymsp[-3].minor.yy184.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy184.offset.n = 0;} break; - case 162: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy400.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy400.offset = yymsp[-1].minor.yy0;} + case 179: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yymsp[-5].minor.yy184.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy184.offset = yymsp[-1].minor.yy0;} break; - case 163: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy400, 0, sizeof(yymsp[1].minor.yy400));} + case 180: /* interval_opt ::= */ +{memset(&yymsp[1].minor.yy184, 0, sizeof(yymsp[1].minor.yy184));} break; - case 164: /* fill_opt ::= */ -{yymsp[1].minor.yy421 = 0; } + case 181: /* session_option ::= */ +{yymsp[1].minor.yy249.col.n = 0; yymsp[1].minor.yy249.gap.n = 0;} break; - case 165: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 182: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + yymsp[-6].minor.yy249.col = yymsp[-4].minor.yy0; + yymsp[-6].minor.yy249.gap = yymsp[-1].minor.yy0; +} + break; + case 183: /* fill_opt ::= */ +{ yymsp[1].minor.yy159 = 0; } + break; + case 184: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy421, &A, -1, 0); - yymsp[-5].minor.yy421 = yymsp[-1].minor.yy421; + tVariantListInsert(yymsp[-1].minor.yy159, &A, -1, 0); + yymsp[-5].minor.yy159 = yymsp[-1].minor.yy159; } break; - case 166: /* fill_opt ::= FILL LP ID RP */ + case 185: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy421 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy159 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 167: /* sliding_opt ::= SLIDING LP tmvar RP */ + case 186: /* sliding_opt ::= SLIDING LP tmvar RP */ {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 168: /* sliding_opt ::= */ + case 187: /* sliding_opt ::= */ {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 169: /* orderby_opt ::= */ -{yymsp[1].minor.yy421 = 0;} + case 189: /* orderby_opt ::= ORDER BY sortlist */ +{yymsp[-2].minor.yy159 = yymsp[0].minor.yy159;} break; - case 170: /* orderby_opt ::= ORDER BY sortlist */ -{yymsp[-2].minor.yy421 = yymsp[0].minor.yy421;} - break; - case 171: /* sortlist ::= sortlist COMMA item sortorder */ + case 190: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy421 = tVariantListAppend(yymsp[-3].minor.yy421, &yymsp[-1].minor.yy430, yymsp[0].minor.yy96); + yylhsminor.yy159 = tVariantListAppend(yymsp[-3].minor.yy159, &yymsp[-1].minor.yy488, yymsp[0].minor.yy20); } - yymsp[-3].minor.yy421 = yylhsminor.yy421; + yymsp[-3].minor.yy159 = yylhsminor.yy159; break; - case 172: /* sortlist ::= item sortorder */ + case 191: /* sortlist ::= item sortorder */ { - yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[-1].minor.yy430, yymsp[0].minor.yy96); + yylhsminor.yy159 = tVariantListAppend(NULL, &yymsp[-1].minor.yy488, yymsp[0].minor.yy20); } - yymsp[-1].minor.yy421 = yylhsminor.yy421; + yymsp[-1].minor.yy159 = yylhsminor.yy159; break; - case 173: /* item ::= ids cpxName */ + case 192: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy430, &yymsp[-1].minor.yy0); + tVariantCreate(&yylhsminor.yy488, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy430 = yylhsminor.yy430; + yymsp[-1].minor.yy488 = yylhsminor.yy488; break; - case 174: /* sortorder ::= ASC */ -{ yymsp[0].minor.yy96 = TSDB_ORDER_ASC; } + case 193: /* sortorder ::= ASC */ +{ yymsp[0].minor.yy20 = TSDB_ORDER_ASC; } break; - case 175: /* sortorder ::= DESC */ -{ yymsp[0].minor.yy96 = TSDB_ORDER_DESC;} + case 194: /* sortorder ::= DESC */ +{ yymsp[0].minor.yy20 = TSDB_ORDER_DESC;} break; - case 176: /* sortorder ::= */ -{ yymsp[1].minor.yy96 = TSDB_ORDER_ASC; } + case 195: /* sortorder ::= */ +{ yymsp[1].minor.yy20 = TSDB_ORDER_ASC; } break; - case 177: /* groupby_opt ::= */ -{ yymsp[1].minor.yy421 = 0;} + case 196: /* groupby_opt ::= */ +{ yymsp[1].minor.yy159 = 0;} break; - case 178: /* groupby_opt ::= GROUP BY grouplist */ -{ yymsp[-2].minor.yy421 = yymsp[0].minor.yy421;} + case 197: /* groupby_opt ::= GROUP BY grouplist */ +{ yymsp[-2].minor.yy159 = yymsp[0].minor.yy159;} break; - case 179: /* grouplist ::= grouplist COMMA item */ + case 198: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy421 = tVariantListAppend(yymsp[-2].minor.yy421, &yymsp[0].minor.yy430, -1); + yylhsminor.yy159 = tVariantListAppend(yymsp[-2].minor.yy159, &yymsp[0].minor.yy488, -1); } - yymsp[-2].minor.yy421 = yylhsminor.yy421; + yymsp[-2].minor.yy159 = yylhsminor.yy159; break; - case 180: /* grouplist ::= item */ + case 199: /* grouplist ::= item */ { - yylhsminor.yy421 = tVariantListAppend(NULL, &yymsp[0].minor.yy430, -1); + yylhsminor.yy159 = tVariantListAppend(NULL, &yymsp[0].minor.yy488, -1); } - yymsp[0].minor.yy421 = yylhsminor.yy421; + yymsp[0].minor.yy159 = yylhsminor.yy159; + break; + case 200: /* having_opt ::= */ + case 210: /* where_opt ::= */ yytestcase(yyruleno==210); + case 250: /* expritem ::= */ yytestcase(yyruleno==250); +{yymsp[1].minor.yy118 = 0;} break; - case 181: /* having_opt ::= */ - case 191: /* where_opt ::= */ yytestcase(yyruleno==191); - case 229: /* expritem ::= */ yytestcase(yyruleno==229); -{yymsp[1].minor.yy78 = 0;} + case 201: /* having_opt ::= HAVING expr */ + case 211: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==211); +{yymsp[-1].minor.yy118 = yymsp[0].minor.yy118;} break; - case 182: /* having_opt ::= HAVING expr */ - case 192: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==192); -{yymsp[-1].minor.yy78 = yymsp[0].minor.yy78;} + case 202: /* limit_opt ::= */ + case 206: /* slimit_opt ::= */ yytestcase(yyruleno==206); +{yymsp[1].minor.yy440.limit = -1; yymsp[1].minor.yy440.offset = 0;} break; - case 183: /* limit_opt ::= */ - case 187: /* slimit_opt ::= */ yytestcase(yyruleno==187); -{yymsp[1].minor.yy167.limit = -1; yymsp[1].minor.yy167.offset = 0;} + case 203: /* limit_opt ::= LIMIT signed */ + case 207: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==207); +{yymsp[-1].minor.yy440.limit = yymsp[0].minor.yy317; yymsp[-1].minor.yy440.offset = 0;} break; - case 184: /* limit_opt ::= LIMIT signed */ - case 188: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==188); -{yymsp[-1].minor.yy167.limit = yymsp[0].minor.yy325; yymsp[-1].minor.yy167.offset = 0;} + case 204: /* limit_opt ::= LIMIT signed OFFSET signed */ +{ yymsp[-3].minor.yy440.limit = yymsp[-2].minor.yy317; yymsp[-3].minor.yy440.offset = yymsp[0].minor.yy317;} break; - case 185: /* limit_opt ::= LIMIT signed OFFSET signed */ -{ yymsp[-3].minor.yy167.limit = yymsp[-2].minor.yy325; yymsp[-3].minor.yy167.offset = yymsp[0].minor.yy325;} + case 205: /* limit_opt ::= LIMIT signed COMMA signed */ +{ yymsp[-3].minor.yy440.limit = yymsp[0].minor.yy317; yymsp[-3].minor.yy440.offset = yymsp[-2].minor.yy317;} break; - case 186: /* limit_opt ::= LIMIT signed COMMA signed */ -{ yymsp[-3].minor.yy167.limit = yymsp[0].minor.yy325; yymsp[-3].minor.yy167.offset = yymsp[-2].minor.yy325;} + case 208: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ +{yymsp[-3].minor.yy440.limit = yymsp[-2].minor.yy317; yymsp[-3].minor.yy440.offset = yymsp[0].minor.yy317;} break; - case 189: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -{yymsp[-3].minor.yy167.limit = yymsp[-2].minor.yy325; yymsp[-3].minor.yy167.offset = yymsp[0].minor.yy325;} + case 209: /* slimit_opt ::= SLIMIT signed COMMA signed */ +{yymsp[-3].minor.yy440.limit = yymsp[0].minor.yy317; yymsp[-3].minor.yy440.offset = yymsp[-2].minor.yy317;} break; - case 190: /* slimit_opt ::= SLIMIT signed COMMA signed */ -{yymsp[-3].minor.yy167.limit = yymsp[0].minor.yy325; yymsp[-3].minor.yy167.offset = yymsp[-2].minor.yy325;} + case 212: /* expr ::= LP expr RP */ +{yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy118->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 193: /* expr ::= LP expr RP */ -{yylhsminor.yy78 = yymsp[-1].minor.yy78; yylhsminor.yy78->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy78->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 213: /* expr ::= ID */ +{ yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 194: /* expr ::= ID */ -{ yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy78 = yylhsminor.yy78; + case 214: /* expr ::= ID DOT ID */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 195: /* expr ::= ID DOT ID */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 215: /* expr ::= ID DOT STAR */ +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 196: /* expr ::= ID DOT STAR */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 216: /* expr ::= INTEGER */ +{ yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 197: /* expr ::= INTEGER */ -{ yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy78 = yylhsminor.yy78; + case 217: /* expr ::= MINUS INTEGER */ + case 218: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==218); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; - case 198: /* expr ::= MINUS INTEGER */ - case 199: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==199); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy78 = yylhsminor.yy78; + case 219: /* expr ::= FLOAT */ +{ yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 200: /* expr ::= FLOAT */ -{ yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy78 = yylhsminor.yy78; + case 220: /* expr ::= MINUS FLOAT */ + case 221: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==221); +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy118 = yylhsminor.yy118; break; - case 201: /* expr ::= MINUS FLOAT */ - case 202: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==202); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy78 = yylhsminor.yy78; + case 222: /* expr ::= STRING */ +{ yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 203: /* expr ::= STRING */ -{ yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy78 = yylhsminor.yy78; + case 223: /* expr ::= NOW */ +{ yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 204: /* expr ::= NOW */ -{ yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy78 = yylhsminor.yy78; + case 224: /* expr ::= VARIABLE */ +{ yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 205: /* expr ::= VARIABLE */ -{ yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy78 = yylhsminor.yy78; + case 225: /* expr ::= BOOL */ +{ yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 206: /* expr ::= BOOL */ -{ yylhsminor.yy78 = tSqlExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy78 = yylhsminor.yy78; + case 226: /* expr ::= NULL */ +{ yylhsminor.yy118 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 207: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy78 = tSqlExprCreateFunction(yymsp[-1].minor.yy166, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy78 = yylhsminor.yy78; + case 227: /* expr ::= ID LP exprlist RP */ +{ yylhsminor.yy118 = tSqlExprCreateFunction(yymsp[-1].minor.yy159, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy118 = yylhsminor.yy118; break; - case 208: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy78 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy78 = yylhsminor.yy78; + case 228: /* expr ::= ID LP STAR RP */ +{ yylhsminor.yy118 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy118 = yylhsminor.yy118; break; - case 209: /* expr ::= expr IS NULL */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, NULL, TK_ISNULL);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 229: /* expr ::= expr IS NULL */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, NULL, TK_ISNULL);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 210: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-3].minor.yy78, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy78 = yylhsminor.yy78; + case 230: /* expr ::= expr IS NOT NULL */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-3].minor.yy118, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy118 = yylhsminor.yy118; break; - case 211: /* expr ::= expr LT expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_LT);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 231: /* expr ::= expr LT expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_LT);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 212: /* expr ::= expr GT expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_GT);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 232: /* expr ::= expr GT expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_GT);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 213: /* expr ::= expr LE expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_LE);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 233: /* expr ::= expr LE expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_LE);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 214: /* expr ::= expr GE expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_GE);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 234: /* expr ::= expr GE expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_GE);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 215: /* expr ::= expr NE expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_NE);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 235: /* expr ::= expr NE expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_NE);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 216: /* expr ::= expr EQ expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_EQ);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 236: /* expr ::= expr EQ expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_EQ);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 217: /* expr ::= expr AND expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_AND);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 237: /* expr ::= expr BETWEEN expr AND expr */ +{ tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy118); yylhsminor.yy118 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy118, yymsp[-2].minor.yy118, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy118, TK_LE), TK_AND);} + yymsp[-4].minor.yy118 = yylhsminor.yy118; break; - case 218: /* expr ::= expr OR expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_OR); } - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 238: /* expr ::= expr AND expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_AND);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 219: /* expr ::= expr PLUS expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_PLUS); } - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 239: /* expr ::= expr OR expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_OR); } + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 220: /* expr ::= expr MINUS expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_MINUS); } - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 240: /* expr ::= expr PLUS expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_PLUS); } + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 221: /* expr ::= expr STAR expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_STAR); } - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 241: /* expr ::= expr MINUS expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_MINUS); } + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 222: /* expr ::= expr SLASH expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_DIVIDE);} - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 242: /* expr ::= expr STAR expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_STAR); } + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 223: /* expr ::= expr REM expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_REM); } - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 243: /* expr ::= expr SLASH expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_DIVIDE);} + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 224: /* expr ::= expr LIKE expr */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-2].minor.yy78, yymsp[0].minor.yy78, TK_LIKE); } - yymsp[-2].minor.yy78 = yylhsminor.yy78; + case 244: /* expr ::= expr REM expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_REM); } + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 225: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy78 = tSqlExprCreate(yymsp[-4].minor.yy78, (tSQLExpr*)yymsp[-1].minor.yy166, TK_IN); } - yymsp[-4].minor.yy78 = yylhsminor.yy78; + case 245: /* expr ::= expr LIKE expr */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-2].minor.yy118, yymsp[0].minor.yy118, TK_LIKE); } + yymsp[-2].minor.yy118 = yylhsminor.yy118; break; - case 226: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy166 = tSqlExprListAppend(yymsp[-2].minor.yy166,yymsp[0].minor.yy78,0);} - yymsp[-2].minor.yy166 = yylhsminor.yy166; + case 246: /* expr ::= expr IN LP exprlist RP */ +{yylhsminor.yy118 = tSqlExprCreate(yymsp[-4].minor.yy118, (tSqlExpr*)yymsp[-1].minor.yy159, TK_IN); } + yymsp[-4].minor.yy118 = yylhsminor.yy118; break; - case 227: /* exprlist ::= expritem */ -{yylhsminor.yy166 = tSqlExprListAppend(0,yymsp[0].minor.yy78,0);} - yymsp[0].minor.yy166 = yylhsminor.yy166; + case 247: /* exprlist ::= exprlist COMMA expritem */ +{yylhsminor.yy159 = tSqlExprListAppend(yymsp[-2].minor.yy159,yymsp[0].minor.yy118,0, 0);} + yymsp[-2].minor.yy159 = yylhsminor.yy159; break; - case 228: /* expritem ::= expr */ -{yylhsminor.yy78 = yymsp[0].minor.yy78;} - yymsp[0].minor.yy78 = yylhsminor.yy78; + case 248: /* exprlist ::= expritem */ +{yylhsminor.yy159 = tSqlExprListAppend(0,yymsp[0].minor.yy118,0, 0);} + yymsp[0].minor.yy159 = yylhsminor.yy159; break; - case 230: /* cmd ::= RESET QUERY CACHE */ -{ setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} + case 249: /* expritem ::= expr */ +{yylhsminor.yy118 = yymsp[0].minor.yy118;} + yymsp[0].minor.yy118 = yylhsminor.yy118; break; - case 231: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 251: /* cmd ::= RESET QUERY CACHE */ +{ setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} + break; + case 252: /* cmd ::= SYNCDB ids REPLICA */ +{ setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &yymsp[-1].minor.yy0);} + break; + case 253: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy159, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 232: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 254: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; toTSDBType(yymsp[0].minor.yy0.type); SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 233: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 255: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy421, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy159, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 234: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 256: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; toTSDBType(yymsp[0].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 235: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 257: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2908,29 +3021,79 @@ static void yy_reduce( toTSDBType(yymsp[0].minor.yy0.type); A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 236: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 258: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy430, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy488, -1); + + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + break; + case 259: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy159, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + break; + case 260: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + + toTSDBType(yymsp[0].minor.yy0.type); + SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); + + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + break; + case 261: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy159, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + break; + case 262: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ +{ + yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; + + toTSDBType(yymsp[0].minor.yy0.type); + SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1); + + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN, TSDB_SUPER_TABLE); + setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); +} + break; + case 263: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ +{ + yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; + + toTSDBType(yymsp[-1].minor.yy0.type); + SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + + toTSDBType(yymsp[0].minor.yy0.type); + A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1); - SAlterTableSQL* pAlterTable = tAlterTableSqlElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-5].minor.yy0, NULL, A, TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 237: /* cmd ::= KILL CONNECTION INTEGER */ + case 264: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 238: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 265: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 239: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 266: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: @@ -2991,7 +3154,7 @@ static void yy_syntax_error( /************ Begin %syntax_error code ****************************************/ pInfo->valid = false; - int32_t outputBufLen = tListLen(pInfo->pzErrMsg); + int32_t outputBufLen = tListLen(pInfo->msg); int32_t len = 0; if(TOKEN.z) { @@ -3001,13 +3164,13 @@ static void yy_syntax_error( if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > outputBufLen) { char tmpstr[128] = {0}; memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); - len = sprintf(pInfo->pzErrMsg, msg, tmpstr); + len = sprintf(pInfo->msg, msg, tmpstr); } else { - len = sprintf(pInfo->pzErrMsg, msg, &TOKEN.z[0]); + len = sprintf(pInfo->msg, msg, &TOKEN.z[0]); } } else { - len = sprintf(pInfo->pzErrMsg, "Incomplete SQL statement"); + len = sprintf(pInfo->msg, "Incomplete SQL statement"); } assert(len <= outputBufLen); diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index c3798b869eb6c02008043346d70b52592239cad0..1856223391ae719ef98492160da9810826b983a9 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index 728de4357ce64abad78eb5bc421ce82a1e1e7a6d..ce7b2f94a177576c8046b299bcb2e695fb5ead2d 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -3,7 +3,7 @@ #include #include -#include "qAst.h" +#include "texpr.h" #include "taosmsg.h" #include "tsdb.h" #include "tskiplist.h" diff --git a/src/query/tests/patternMatchTest.cpp b/src/query/tests/patternMatchTest.cpp index fa2b58a10bd044491a2fabbaee6180c37320ae35..f3e0d3e119259d0a7cc8a94a6b43f4c71558bf78 100644 --- a/src/query/tests/patternMatchTest.cpp +++ b/src/query/tests/patternMatchTest.cpp @@ -3,7 +3,7 @@ #include #include -#include "tsqlfunction.h" +#include "qAggMain.h" #include "tcompare.h" TEST(testCase, patternMatchTest) { diff --git a/src/query/tests/percentileTest.cpp b/src/query/tests/percentileTest.cpp index f1fc45850138ba287b4597c51c4bb5220626ad8b..104bfb3c06a9613bafcd4e0b3f39af4f9d102b04 100644 --- a/src/query/tests/percentileTest.cpp +++ b/src/query/tests/percentileTest.cpp @@ -43,12 +43,12 @@ tMemBucket *createDoubleDataBucket(int32_t start, int32_t end) { } tMemBucket *createUnsignedDataBucket(int32_t start, int32_t end, int32_t type) { - tMemBucket *pBucket = tMemBucketCreate(tDataTypeDesc[type].nSize, type, start, end); + tMemBucket *pBucket = tMemBucketCreate(tDataTypes[type].bytes, type, start, end); for (int32_t i = start; i <= end; ++i) { uint64_t k = i; int32_t ret = tMemBucketPut(pBucket, &k, 1); if (ret != 0) { - printf("value out of range:%f", k); + printf("value out of range:%" PRId64, k); } } @@ -245,7 +245,7 @@ void unsignedDataTest() { } // namespace TEST(testCase, percentileTest) { - // qsortTest(); +// qsortTest(); intDataTest(); bigintDataTest(); doubleDataTest(); diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 7b946d858913b14dfdbb71eb1b8a8c469e6d930c..19e011774241dffa156a0ecb003041aba8253ba5 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -10,7 +10,7 @@ namespace { // simple test void simpleTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4096, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, NULL); int32_t pageId = 0; int32_t groupId = 0; @@ -52,7 +52,7 @@ void simpleTest() { void writeDownTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, NULL); int32_t pageId = 0; int32_t writePageId = 0; @@ -99,7 +99,7 @@ void writeDownTest() { void recyclePageTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, NULL); int32_t pageId = 0; int32_t writePageId = 0; diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index a8500364dceaef97d215b72ef805163e726bdaf9..3406d8309023118d16cfc4dfbdab58defc6f005a 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -227,10 +227,10 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_SQL); char t61[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_SQL); char t61_1[] = "' ABC '"; - EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_SQL); char t62[] = " ABC . def "; EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_SQL); @@ -249,13 +249,13 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_SQL); char t66[] = "' ABC '.' DEF '"; - EXPECT_EQ(testValidateName(t66), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_SQL); char t67[] = "abc . ' DEF '"; EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_SQL); char t68[] = "' abc '.' DEF '"; - EXPECT_EQ(testValidateName(t68), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_SQL); // do not use key words char t69[] = "table.'DEF'"; @@ -265,7 +265,7 @@ TEST(testCase, db_table_name) { EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_SQL); char t71[] = "'_abXYZ1234 '.' deFF '"; - EXPECT_EQ(testValidateName(t71), TSDB_CODE_SUCCESS); + EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_SQL); char t72[] = "'_abDEF&^%1234'.' DIef'"; EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_SQL); diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 02a1e7c2d81d13194ca819c703657df5e1f3fb3b..f94b4aeb6d21277b6b845587cd35a2c98e0bc0b0 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 17b80a0afb6f175a73dce5cb361674030561c96e..133ae6d0abbc77e141c6aef7835428b841c4988e 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -246,8 +246,12 @@ void *rpcOpen(const SRpcInit *pInit) { if(pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); pRpc->connType = pInit->connType; + if (pRpc->connType == TAOS_CONN_CLIENT) { + pRpc->numOfThreads = pInit->numOfThreads; + } else { + pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; + } pRpc->idleTime = pInit->idleTime; - pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; pRpc->localPort = pInit->localPort; pRpc->afp = pInit->afp; pRpc->sessions = pInit->sessions+1; @@ -1013,6 +1017,13 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont return pConn; } +static void doRpcReportBrokenLinkToServer(void *param, void *id) { + SRpcMsg *pRpcMsg = (SRpcMsg *)(param); + SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); + SRpcInfo *pRpc = pConn->pRpc; + (*(pRpc->cfp))(pRpcMsg, NULL); + free(pRpcMsg); +} static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { SRpcInfo *pRpc = pConn->pRpc; if (pConn->pReqMsg == NULL) return; @@ -1021,16 +1032,20 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { rpcAddRef(pRpc); tDebug("%s, notify the server app, connection is gone", pConn->info); - SRpcMsg rpcMsg; - rpcMsg.pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server - rpcMsg.contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length - rpcMsg.ahandle = pConn->ahandle; - rpcMsg.handle = pConn; - rpcMsg.msgType = pConn->inType; - rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + SRpcMsg *rpcMsg = malloc(sizeof(SRpcMsg)); + rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server + rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length + rpcMsg->ahandle = pConn->ahandle; + rpcMsg->handle = pConn; + rpcMsg->msgType = pConn->inType; + rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pReqMsg = NULL; pConn->reqMsgLen = 0; - if (pRpc->cfp) (*(pRpc->cfp))(&rpcMsg, NULL); + if (pRpc->cfp) { + taosTmrStart(doRpcReportBrokenLinkToServer, 0, rpcMsg, pRpc->tmrCtrl); + } else { + free(rpcMsg); + } } static void rpcProcessBrokenLink(SRpcConn *pConn) { @@ -1047,7 +1062,7 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { pConn->pReqMsg = NULL; taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); } - + if (pConn->inType) rpcReportBrokenLinkToServer(pConn); rpcReleaseConn(pConn); @@ -1277,7 +1292,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { SRpcConn *pConn = rpcSetupConnToServer(pContext); if (pConn == NULL) { pContext->code = terrno; - taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); + taosTmrStart(rpcProcessConnError, 1, pContext, pRpc->tmrCtrl); return; } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index daf3bd86c168ea75c29f5229cecea1acd0c66cd3..3162ab2e4cbabc2a1ce9ed863589e3be0e92f75f 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -17,25 +17,18 @@ #include "tsocket.h" #include "tutil.h" #include "taosdef.h" -#include "taoserror.h" +#include "taoserror.h" #include "rpcLog.h" #include "rpcHead.h" #include "rpcTcp.h" -#ifdef WINDOWS -#include "wepoll.h" -#endif - -#ifndef EPOLLWAKEUP - #define EPOLLWAKEUP (1u << 29) -#endif typedef struct SFdObj { void *signature; SOCKET fd; // TCP socket FD - int closedByApp; // 1: already closed by App void *thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; + int16_t closedByApp; // 1: already closed by App struct SThreadObj *pThreadObj; struct SFdObj *prev; struct SFdObj *next; @@ -47,7 +40,7 @@ typedef struct SThreadObj { pthread_mutex_t mutex; uint32_t ip; bool stop; - SOCKET pollFd; + EpollFd pollFd; int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; @@ -55,10 +48,19 @@ typedef struct SThreadObj { void *(*processData)(SRecvInfo *pPacket); } SThreadObj; +typedef struct { + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; + SThreadObj **pThreadObj; +} SClientObj; + typedef struct { SOCKET fd; uint32_t ip; uint16_t port; + int8_t stop; + int8_t reserve; char label[TSDB_LABEL_LEN]; int numOfThreads; void * shandle; @@ -79,7 +81,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pServerObj = (SServerObj *)calloc(sizeof(SServerObj), 1); if (pServerObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -93,7 +95,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pServerObj->pThreadObj = (SThreadObj **)calloc(sizeof(SThreadObj *), numOfThreads); if (pServerObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); free(pServerObj); return NULL; } @@ -103,24 +105,25 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - // initialize parameters in case it may encounter error later + // initialize parameters in case it may encounter error later for (int i = 0; i < numOfThreads; ++i) { pThreadObj = (SThreadObj *)calloc(sizeof(SThreadObj), 1); if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); for (int j=0; jpThreadObj[j]); free(pServerObj->pThreadObj); free(pServerObj); return NULL; } - + pServerObj->pThreadObj[i] = pThreadObj; pThreadObj->pollFd = -1; taosResetPthread(&pThreadObj->thread); pThreadObj->processData = fp; tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); pThreadObj->shandle = shandle; + pThreadObj->stop = false; } // initialize mutex, thread, fd which may fail @@ -132,7 +135,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread break; } - pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter + pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThreadObj->pollFd < 0) { tError("%s failed to create TCP epoll", label); code = -1; @@ -149,17 +152,17 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port); - if (pServerObj->fd < 0) code = -1; + if (pServerObj->fd < 0) code = -1; - if (code == 0) { + if (code == 0) { code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj); if (code != 0) { - tError("%s failed to create TCP accept thread(%s)", label, strerror(errno)); + tError("%s failed to create TCP accept thread(%s)", label, strerror(code)); } } if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); taosCleanUpTcpServer(pServerObj); pServerObj = NULL; } else { @@ -171,8 +174,9 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread } static void taosStopTcpThread(SThreadObj* pThreadObj) { - // save thread into local variable and signal thread to stop - pthread_t thread = pThreadObj->thread; + if (pThreadObj == NULL) { return;} + // save thread into local variable and signal thread to stop + pthread_t thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { return; } @@ -188,8 +192,20 @@ void taosStopTcpServer(void *handle) { SServerObj *pServerObj = handle; if (pServerObj == NULL) return; - if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD); + pServerObj->stop = 1; + if (pServerObj->fd >= 0) { +#ifdef WINDOWS + closesocket(pServerObj->fd); +#elif defined(__APPLE__) + if (pServerObj->fd!=-1) { + close(pServerObj->fd); + pServerObj->fd = -1; + } +#else + shutdown(pServerObj->fd, SHUT_RD); +#endif + } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { pthread_detach(pthread_self()); @@ -230,6 +246,11 @@ static void *taosAcceptTcpConnection(void *arg) { while (1) { socklen_t addrlen = sizeof(caddr); connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); + if (pServerObj->stop) { + tDebug("%s TCP server stop accepting new connections", pServerObj->label); + break; + } + if (connFd == -1) { if (errno == EINVAL) { tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label); @@ -249,7 +270,7 @@ static void *taosAcceptTcpConnection(void *arg) { taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); continue; } - + // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; @@ -258,13 +279,13 @@ static void *taosAcceptTcpConnection(void *arg) { if (pFdObj) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); - tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, + tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); - } + } // pick up next thread for next connection threadId++; @@ -275,70 +296,107 @@ static void *taosAcceptTcpConnection(void *arg) { return NULL; } -void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle) { - SThreadObj *pThreadObj; - pthread_attr_t thattr; - - pThreadObj = (SThreadObj *)malloc(sizeof(SThreadObj)); - memset(pThreadObj, 0, sizeof(SThreadObj)); - tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); - pThreadObj->ip = ip; - pThreadObj->shandle = shandle; - - if (pthread_mutex_init(&(pThreadObj->mutex), NULL) < 0) { - tError("%s failed to init TCP client mutex(%s)", label, strerror(errno)); - free(pThreadObj); - terrno = TAOS_SYSTEM_ERROR(errno); +void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { + SClientObj *pClientObj = (SClientObj *)calloc(1, sizeof(SClientObj)); + if (pClientObj == NULL) { + tError("TCP:%s no enough memory", label); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - pThreadObj->pollFd = (SOCKET)epoll_create(10); // size does not matter - if (pThreadObj->pollFd < 0) { - tError("%s failed to create TCP client epoll", label); - free(pThreadObj); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; + tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); + pClientObj->numOfThreads = numOfThreads; + pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*)); + if (pClientObj->pThreadObj == NULL) { + tError("TCP:%s no enough memory", label); + tfree(pClientObj); + terrno = TAOS_SYSTEM_ERROR(errno); } - pThreadObj->processData = fp; - + int code = 0; + pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); - pthread_attr_destroy(&thattr); - if (code != 0) { - taosCloseSocket(pThreadObj->pollFd); - free(pThreadObj); - terrno = TAOS_SYSTEM_ERROR(errno); - tError("%s failed to create TCP read data thread(%s)", label, strerror(errno)); - return NULL; + + for (int i = 0; i < numOfThreads; ++i) { + SThreadObj *pThreadObj = (SThreadObj *)calloc(1, sizeof(SThreadObj)); + if (pThreadObj == NULL) { + tError("TCP:%s no enough memory", label); + terrno = TAOS_SYSTEM_ERROR(errno); + for (int j=0; jpThreadObj[j]); + free(pClientObj); + pthread_attr_destroy(&thattr); + return NULL; + } + pClientObj->pThreadObj[i] = pThreadObj; + taosResetPthread(&pThreadObj->thread); + pThreadObj->ip = ip; + pThreadObj->stop = false; + tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); + pThreadObj->shandle = shandle; + pThreadObj->processData = fp; } - tDebug("%s TCP client is initialized, ip:%u:%hu", label, ip, port); + // initialize mutex, thread, fd which may fail + for (int i = 0; i < numOfThreads; ++i) { + SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; + code = pthread_mutex_init(&(pThreadObj->mutex), NULL); + if (code < 0) { + tError("%s failed to init TCP process data mutex(%s)", label, strerror(errno)); + break; + } - return pThreadObj; + pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter + if (pThreadObj->pollFd < 0) { + tError("%s failed to create TCP epoll", label); + code = -1; + break; + } + + code = pthread_create(&(pThreadObj->thread), &thattr, taosProcessTcpData, (void *)(pThreadObj)); + if (code != 0) { + tError("%s failed to create TCP process data thread(%s)", label, strerror(errno)); + break; + } + pThreadObj->threadId = i; + } + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosCleanUpTcpClient(pClientObj); + pClientObj = NULL; + } + return pClientObj; } void taosStopTcpClient(void *chandle) { - SThreadObj *pThreadObj = chandle; - if (pThreadObj == NULL) return; + SClientObj *pClientObj = chandle; - tDebug ("%s TCP client is stopped", pThreadObj->label); + if (pClientObj == NULL) return; + + tDebug ("%s TCP client is stopped", pClientObj->label); } void taosCleanUpTcpClient(void *chandle) { - SThreadObj *pThreadObj = chandle; - if (pThreadObj == NULL) return; + SClientObj *pClientObj = chandle; + if (pClientObj == NULL) return; + for (int i = 0; i < pClientObj->numOfThreads; ++i) { + SThreadObj *pThreadObj= pClientObj->pThreadObj[i]; + taosStopTcpThread(pThreadObj); + } - tDebug ("%s TCP client will be cleaned up", pThreadObj->label); - taosStopTcpThread(pThreadObj); + tDebug("%s TCP client is cleaned up", pClientObj->label); + tfree(pClientObj->pThreadObj); + tfree(pClientObj); } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { - SThreadObj * pThreadObj = shandle; + SClientObj * pClientObj = shandle; + int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; + atomic_store_32(&pClientObj->index, index + 1); + SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); - if (fd < 0) return NULL; + if (fd <= 0) return NULL; struct sockaddr_in sin; uint16_t localPort = 0; @@ -349,12 +407,12 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin } SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); - + if (pFdObj) { pFdObj->thandle = thandle; pFdObj->port = port; pFdObj->ip = ip; - tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", + tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); @@ -369,7 +427,7 @@ void taosCloseTcpConnection(void *chandle) { if (pFdObj == NULL || pFdObj->signature != pFdObj) return; SThreadObj *pThreadObj = pFdObj->pThreadObj; - tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); + tDebug("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); // pFdObj->thandle = NULL; pFdObj->closedByApp = 1; @@ -382,7 +440,7 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand SThreadObj *pThreadObj = pFdObj->pThreadObj; int ret = taosWriteMsg(pFdObj->fd, data, len); - tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); + tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); return ret; } @@ -405,7 +463,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { recvInfo.chandle = NULL; recvInfo.connType = RPC_CONN_TCP; (*(pThreadObj->processData))(&recvInfo); - } + } taosFreeFdObj(pFdObj); } @@ -420,7 +478,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); - return -1; + return -1; } msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); @@ -438,14 +496,14 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s %p read error, leftLen:%d retLen:%d FD:%p", + tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); free(buffer); return -1; } memcpy(msg, &rpcHead, sizeof(SRpcHead)); - + pInfo->msg = msg; pInfo->msgLen = msgLen; pInfo->ip = pFdObj->ip; @@ -456,7 +514,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { pInfo->connType = RPC_CONN_TCP; if (pFdObj->closedByApp) { - free(buffer); + free(buffer); return -1; } @@ -470,7 +528,10 @@ static void *taosProcessTcpData(void *param) { SFdObj *pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; - + +#ifdef __APPLE__ + taos_block_sigalrm(); +#endif // __APPLE__ while (1) { int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThreadObj->stop) { @@ -501,7 +562,7 @@ static void *taosProcessTcpData(void *param) { } if (taosReadTcpData(pFdObj, &recvInfo) < 0) { - shutdown(pFdObj->fd, SHUT_WR); + shutdown(pFdObj->fd, SHUT_WR); continue; } @@ -509,10 +570,13 @@ static void *taosProcessTcpData(void *param) { if (pFdObj->thandle == NULL) taosFreeFdObj(pFdObj); } - if (pThreadObj->stop) break; + if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >=0) taosCloseSocket(pThreadObj->pollFd); + if (pThreadObj->pollFd >=0) { + EpollClose(pThreadObj->pollFd); + pThreadObj->pollFd = -1; + } while (pThreadObj->pHead) { SFdObj *pFdObj = pThreadObj->pHead; @@ -544,7 +608,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { tfree(pFdObj); - terrno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -578,7 +642,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s %p TCP thread:%d, number of FDs is negative!!!", + tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); if (pFdObj->prev) { @@ -593,7 +657,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 22301fcecc83fb1f4c9a29132b0311e05b2382e6..7a46dbe5c3e1238da93043f8dc97047f84ff72e8 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -15,7 +15,6 @@ #include "os.h" #include "tsocket.h" -#include "tsystem.h" #include "ttimer.h" #include "tutil.h" #include "taosdef.h" @@ -140,6 +139,7 @@ void taosStopUdpConnection(void *handle) { pConn = pSet->udpConn + i; if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR); if (pConn->fd >=0) taosCloseSocket(pConn->fd); + pConn->fd = -1; } for (int i = 0; i < pSet->threads; ++i) { @@ -197,9 +197,16 @@ static void *taosRecvUdpData(void *param) { while (1) { dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); - if(dataLen <= 0) { - tDebug("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno)); - break; + if (dataLen <= 0) { + tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d fd:%d", pConn->label, strerror(errno), (int32_t)dataLen, + pConn->fd); + + // for windows usage, remote shutdown also returns - 1 in windows client + if (pConn->fd == -1) { + break; + } else { + continue; + } } port = ntohs(sourceAdd.sin_port); diff --git a/src/rpc/test/CMakeLists.txt b/src/rpc/test/CMakeLists.txt index 9a4bcc353d9bd1ad8ee8b6ef546f8e4a44f75559..c10cea6c9dd8c53ab8608c8a736795f2318059d8 100644 --- a/src/rpc/test/CMakeLists.txt +++ b/src/rpc/test/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) @@ -16,3 +16,17 @@ IF (TD_LINUX) ADD_EXECUTABLE(rserver ${SERVER_SRC}) TARGET_LINK_LIBRARIES(rserver trpc) ENDIF () + +IF (TD_DARWIN) + LIST(APPEND CLIENT_SRC ./rclient.c) + ADD_EXECUTABLE(rclient ${CLIENT_SRC}) + TARGET_LINK_LIBRARIES(rclient trpc) + + LIST(APPEND SCLIENT_SRC ./rsclient.c) + ADD_EXECUTABLE(rsclient ${SCLIENT_SRC}) + TARGET_LINK_LIBRARIES(rsclient trpc) + + LIST(APPEND SERVER_SRC ./rserver.c) + ADD_EXECUTABLE(rserver ${SERVER_SRC}) + TARGET_LINK_LIBRARIES(rserver trpc) +ENDIF () diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index 6a533808415929776be46a96266dd7cdaeca0bf3..82d0bbf520843f5418d1004f2fe7c1be756b7b6f 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -1,18 +1,16 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) - -IF (TD_LINUX) - LIST(REMOVE_ITEM SRC src/syncArbitrator.c) - ADD_LIBRARY(sync ${SRC}) - TARGET_LINK_LIBRARIES(sync tutil pthread common) - LIST(APPEND BIN_SRC src/syncArbitrator.c) - LIST(APPEND BIN_SRC src/syncTcp.c) - ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) - TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil) +LIST(REMOVE_ITEM SRC src/syncArbitrator.c) +ADD_LIBRARY(sync ${SRC}) +TARGET_LINK_LIBRARIES(sync tutil pthread common) - #ADD_SUBDIRECTORY(test) -ENDIF () +LIST(APPEND BIN_SRC src/syncArbitrator.c) +LIST(APPEND BIN_SRC src/syncTcp.c) +ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) +TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil) + +#ADD_SUBDIRECTORY(test) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index eef687d64753bdecfddaf63759ba1d3bdd8fa2bd..91613ae35107f0a667798e730f6021687e3ced5e 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -82,8 +82,8 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process uint64_t lastFileVer; // track the file version while retrieve uint64_t lastWalVer; // track the wal version while retrieve - int32_t syncFd; - int32_t peerFd; // forward FD + SOCKET syncFd; + SOCKET peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process int32_t refCount; @@ -108,14 +108,17 @@ typedef struct SSyncNode { SSyncFwds * pSyncFwds; // saved forward info if quorum >1 void * pFwdTimer; void * pRoleTimer; - FGetFileInfo getFileInfo; - FGetWalInfo getWalInfo; - FWriteToCache writeToCache; + void * pTsdb; + FGetWalInfo getWalInfoFp; + FWriteToCache writeToCacheFp; FConfirmForward confirmForward; - FNotifyRole notifyRole; - FNotifyFlowCtrl notifyFlowCtrl; - FNotifyFileSynced notifyFileSynced; - FGetVersion getVersion; + FNotifyRole notifyRoleFp; + FNotifyFlowCtrl notifyFlowCtrlFp; + FStartSyncFile startSyncFileFp; + FStopSyncFile stopSyncFileFp; + FGetVersion getVersionFp; + FSendFile sendFileFp; + FRecvFile recvFileFp; pthread_mutex_t mutex; } SSyncNode; diff --git a/src/sync/inc/syncMsg.h b/src/sync/inc/syncMsg.h index 73f4223c882ff9ce544984b37843070cf579c44a..85ac9c78affa5282d5ca703caffc1bc5c24461bb 100644 --- a/src/sync/inc/syncMsg.h +++ b/src/sync/inc/syncMsg.h @@ -37,7 +37,8 @@ typedef enum { TAOS_SMSG_SETUP_RSP = 12, TAOS_SMSG_SYNC_FILE = 13, TAOS_SMSG_SYNC_FILE_RSP = 14, - TAOS_SMSG_END = 15, + TAOS_SMSG_TEST = 15, + TAOS_SMSG_END = 16 } ESyncMsgType; typedef enum { @@ -98,16 +99,12 @@ typedef struct { typedef struct { SSyncHead head; - char name[TSDB_FILENAME_LEN]; - uint32_t magic; - uint32_t index; uint64_t fversion; - int64_t size; -} SFileInfo; +} SFileVersion; typedef struct { SSyncHead head; - int8_t sync; + int8_t ack; } SFileAck; typedef struct { @@ -132,9 +129,10 @@ void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); -void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId); +void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId); #ifdef __cplusplus } diff --git a/src/sync/inc/syncTcp.h b/src/sync/inc/syncTcp.h index d4674fee6bd07d5ab68284399d9a6502dbd5ec2c..b322c3440c4eeb5b53b1d55a2446030b4f892e0b 100644 --- a/src/sync/inc/syncTcp.h +++ b/src/sync/inc/syncTcp.h @@ -27,12 +27,12 @@ typedef struct { int32_t bufferSize; void (*processBrokenLink)(int64_t handleId); int32_t (*processIncomingMsg)(int64_t handleId, void *buffer); - void (*processIncomingConn)(int32_t fd, uint32_t ip); + void (*processIncomingConn)(SOCKET fd, uint32_t ip); } SPoolInfo; void *syncOpenTcpThreadPool(SPoolInfo *pInfo); void syncCloseTcpThreadPool(void *); -void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd); +void *syncAllocateTcpConn(void *, int64_t rid, SOCKET connFd); void syncFreeTcpConn(void *); #ifdef __cplusplus diff --git a/src/sync/src/syncArbitrator.c b/src/sync/src/syncArbitrator.c index fed0774346693fbd2d5ef725bfa620433664cee8..fdbef4c9f827d77bdc1ce1874dc2b368a7467ca1 100644 --- a/src/sync/src/syncArbitrator.c +++ b/src/sync/src/syncArbitrator.c @@ -27,8 +27,9 @@ #include "syncInt.h" #include "syncTcp.h" -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +extern void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd); +static void arbSignalHandler(int32_t signum, void *sigInfo, void *context); +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void arbProcessBrokenLink(int64_t rid); static int32_t arbProcessPeerMsg(int64_t rid, void *buffer); static tsem_t tsArbSem; @@ -36,7 +37,7 @@ static void * tsArbTcpPool; typedef struct { char id[TSDB_EP_LEN + 24]; - int32_t nodeFd; + SOCKET nodeFd; void * pConn; } SNodeConn; @@ -69,12 +70,10 @@ int32_t main(int32_t argc, char *argv[]) { } /* Set termination handler. */ - struct sigaction act = {{0}}; - act.sa_flags = SA_SIGINFO; - act.sa_sigaction = arbSignalHandler; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); + taosSetSignal(SIGTERM, arbSignalHandler); + taosSetSignal(SIGINT, arbSignalHandler); + taosSetSignal(SIGHUP, arbSignalHandler); + taosSetSignal(SIGABRT, arbSignalHandler); tsAsyncLog = 0; strcat(arbLogPath, "/arbitrator.log"); @@ -103,12 +102,12 @@ int32_t main(int32_t argc, char *argv[]) { syncCloseTcpThreadPool(tsArbTcpPool); sInfo("TAOS arbitrator is shut down"); - closelog(); + closelog(); return 0; } -static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); @@ -120,6 +119,11 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } + if (msg.head.type == TAOS_SMSG_TEST) { + syncProcessTestMsg(&msg, connFd); + return; + } + SNodeConn *pNode = calloc(sizeof(SNodeConn), 1); if (pNode == NULL) { sError("failed to allocate memory since %s", strerror(errno)); @@ -172,14 +176,13 @@ static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) { return 0; } -static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) { - struct sigaction act = {{0}}; - act.sa_handler = SIG_IGN; - sigaction(SIGTERM, &act, NULL); - sigaction(SIGHUP, &act, NULL); - sigaction(SIGINT, &act, NULL); +static void arbSignalHandler(int32_t signum, void *sigInfo, void *context) { + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGHUP); - sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); + sInfo("shut down signal is %d", signum); // inform main thread to exit tsem_post(&tsArbSem); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 98100fbdd87dddc3d9d30431db912813172405b9..d21743d40ab43b90a7654a8e0c10ca49e2d62cd8 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -45,7 +45,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId); static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId); static void syncProcessBrokenLink(int64_t rid); static int32_t syncProcessPeerMsg(int64_t rid, void *buffer); -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp); static void syncRemovePeer(SSyncPeer *pPeer); static void syncAddArbitrator(SSyncNode *pNode); static void syncFreeNode(void *); @@ -56,7 +56,7 @@ static void syncMonitorNodeRole(void *param, void *tmrId); static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); static void syncRestartPeer(SSyncPeer *pPeer); -static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp); +static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype, bool force); static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); static void syncStartCheckPeerConn(SSyncPeer *pPeer); @@ -174,19 +174,22 @@ int64_t syncStart(const SSyncInfo *pInfo) { tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path)); pthread_mutex_init(&pNode->mutex, NULL); - pNode->getFileInfo = pInfo->getFileInfo; - pNode->getWalInfo = pInfo->getWalInfo; - pNode->writeToCache = pInfo->writeToCache; - pNode->notifyRole = pInfo->notifyRole; + pNode->getWalInfoFp = pInfo->getWalInfoFp; + pNode->writeToCacheFp = pInfo->writeToCacheFp; + pNode->notifyRoleFp = pInfo->notifyRoleFp; pNode->confirmForward = pInfo->confirmForward; - pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl; - pNode->notifyFileSynced = pInfo->notifyFileSynced; - pNode->getVersion = pInfo->getVersion; + pNode->notifyFlowCtrlFp = pInfo->notifyFlowCtrlFp; + pNode->startSyncFileFp = pInfo->startSyncFileFp; + pNode->stopSyncFileFp = pInfo->stopSyncFileFp; + pNode->getVersionFp = pInfo->getVersionFp; + pNode->sendFileFp = pInfo->sendFileFp; + pNode->recvFileFp = pInfo->recvFileFp; pNode->selfIndex = -1; pNode->vgId = pInfo->vgId; pNode->replica = pCfg->replica; pNode->quorum = pCfg->quorum; + pNode->pTsdb = pInfo->pTsdb; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; pNode->refCount = 1; @@ -248,8 +251,8 @@ int64_t syncStart(const SSyncInfo *pInfo) { syncAddArbitrator(pNode); taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *)); - if (pNode->notifyRole) { - (*pNode->notifyRole)(pNode->vgId, nodeRole); + if (pNode->notifyRoleFp) { + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb @@ -357,7 +360,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { if (pNewCfg->replica <= 1) { sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId); nodeRole = TAOS_SYNC_ROLE_MASTER; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]); // arb @@ -374,24 +377,24 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { return 0; } -int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) { +int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype, bool force) { if (rid <= 0) return 0; SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return 0; - int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); + int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype, force); syncReleaseNode(pNode); return code; } -void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force) { SSyncNode *pNode = syncAcquireNode(rid); if (pNode == NULL) return; SSyncPeer *pPeer = pNode->pMaster; - if (pPeer && pNode->quorum > 1) { + if (pPeer && (pNode->quorum > 1 || force)) { SFwdRsp rsp; syncBuildSyncFwdRsp(&rsp, pNode->vgId, version, code); @@ -406,7 +409,7 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { syncReleaseNode(pNode); } -#if 0 +#if 1 void syncRecover(int64_t rid) { SSyncPeer *pPeer; @@ -417,7 +420,7 @@ void syncRecover(int64_t rid) { // if take this node to unsync state, the whole system may not work nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); nodeVersion = 0; pthread_mutex_lock(&pNode->mutex); @@ -544,7 +547,7 @@ static void syncClosePeerConn(SSyncPeer *pPeer) { sDebug("%s, pfd:%d sfd:%d will be closed", pPeer->id, pPeer->peerFd, pPeer->syncFd); taosTmrStopA(&pPeer->timer); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; void *pConn = pPeer->pConn; @@ -568,7 +571,7 @@ static void syncStartCheckPeerConn(SSyncPeer *pPeer) { int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { int32_t checkMs = 100 + (pNode->vgId * 10) % 100; - if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs; + sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs); taosTmrReset(syncCheckPeerConnection, checkMs, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } @@ -625,8 +628,8 @@ static void syncResetFlowCtrl(SSyncNode *pNode) { pNode->peerInfo[index]->numOfRetrieves = 0; } - if (pNode->notifyFlowCtrl) { - (*pNode->notifyFlowCtrl)(pNode->vgId, 0); + if (pNode->notifyFlowCtrlFp) { + (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); } } @@ -694,7 +697,7 @@ static void syncChooseMaster(SSyncNode *pNode) { taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER); syncResetFlowCtrl(pNode); - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } else { pPeer = pNode->peerInfo[index]; sInfo("%s, it shall work as master", pPeer->id); @@ -730,7 +733,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } } else { for (int32_t index = 0; index < pNode->replica; ++index) { @@ -738,11 +741,14 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; if (masterIndex < 0) { masterIndex = index; + sDebug("vgId:%d, peer:%s is master, index:%d", pNode->vgId, pTemp->id, index); } else { // multiple masters, it shall not happen if (masterIndex == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); + } else { + sError("vgId:%d, peer:%s is master too, masterIndex:%d index:%d", pNode->vgId, pTemp->id, masterIndex, index); } } } @@ -759,7 +765,7 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); code = -1; for (int32_t index = 0; index < pNode->replica; ++index) { @@ -796,7 +802,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } else { sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); } } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); @@ -830,7 +836,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } if (oldPeerRole != newPeerRole || nodeRole != oldSelfRole) { - sDebug("vgId:%d, roles changed, broadcast status", pNode->vgId); + sDebug("vgId:%d, roles changed, broadcast status, replica:%d", pNode->vgId, pNode->replica); syncBroadcastStatus(pNode); } @@ -857,8 +863,12 @@ static void syncRestartPeer(SSyncPeer *pPeer) { void syncRestartConnection(SSyncPeer *pPeer) { if (pPeer->ip == 0) return; + if (syncAcquirePeer(pPeer->rid) == NULL) return; + syncRestartPeer(pPeer); syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE); + + syncReleasePeer(pPeer); } static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { @@ -869,7 +879,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { if (nodeRole != TAOS_SYNC_ROLE_MASTER) { sError("%s, I am not master anymore", pPeer->id); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); return; } @@ -989,7 +999,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; - (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); + (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); } else { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); @@ -1114,8 +1124,8 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); - if (connFd < 0) { + SOCKET connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + if (connFd <= 0) { sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno)); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); return; @@ -1132,7 +1142,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { if (pPeer->isArb) tsArbOnline = 1; } else { sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno)); - taosClose(connFd); + taosCloseSocket(connFd); taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer); } } @@ -1171,7 +1181,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_INIT; sError("%s, failed to create sync restore thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); syncReleasePeer(pPeer); } else { sInfo("%s, sync restore thread:0x%08" PRIx64 " create successfully, rid:%" PRId64, pPeer->id, @@ -1179,7 +1189,21 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { +void syncProcessTestMsg(SSyncMsg *pMsg, SOCKET connFd) { + sInfo("recv sync test msg"); + + SSyncMsg rsp; + syncBuildSyncTestMsg(&rsp, -1); + if (taosWriteMsg(connFd, &rsp, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + sInfo("failed to send sync test rsp since %s", strerror(errno)); + } + + sInfo("send sync test rsp"); + taosMsleep(1000); + taosCloseSocket(connFd); +} + +static void syncProcessIncommingConnection(SOCKET connFd, uint32_t sourceIp) { char ipstr[24]; int32_t i; @@ -1200,6 +1224,11 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { return; } + if (msg.head.type == TAOS_SMSG_TEST) { + syncProcessTestMsg(&msg, connFd); + return; + } + int32_t vgId = msg.head.vgId; SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { @@ -1384,7 +1413,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { syncReleaseNode(pNode); } -static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) { +static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype, bool force) { SSyncPeer *pPeer; SSyncHead *pSyncHead; SWalHead * pWalHead = data; @@ -1428,7 +1457,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle if (pPeer == NULL || pPeer->peerFd < 0) continue; if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; - if (pNode->quorum > 1 && code == 0) { + if ((pNode->quorum > 1 || force) && code == 0) { code = syncSaveFwdInfo(pNode, pWalHead->version, mhandle); if (code >= 0) code = 1; } diff --git a/src/sync/src/syncMsg.c b/src/sync/src/syncMsg.c index 034f9a98a70c7373b74a84d24d478b52d7bf4df9..3348f1ec337e1b7c292d231360765c914936625c 100644 --- a/src/sync/src/syncMsg.c +++ b/src/sync/src/syncMsg.c @@ -86,6 +86,7 @@ static void syncBuildMsg(SSyncMsg *pMsg, int32_t vgId, ESyncMsgType type) { void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_REQ); } void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SYNC_DATA); } void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_SETUP); } +void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId) { syncBuildMsg(pMsg, vgId, TAOS_SMSG_TEST); } void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId) { pMsg->head.type = TAOS_SMSG_STATUS; @@ -101,9 +102,9 @@ void syncBuildFileAck(SFileAck *pMsg, int32_t vgId) { syncBuildHead(&pMsg->head); } -void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId) { +void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId) { pMsg->head.type = TAOS_SMSG_SYNC_FILE; pMsg->head.vgId = vgId; - pMsg->head.len = sizeof(SFileInfo) - sizeof(SSyncHead); + pMsg->head.len = sizeof(SFileVersion) - sizeof(SSyncHead); syncBuildHead(&pMsg->head); } \ No newline at end of file diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 78520c660875ccd25d42857b89359fbd15f33b35..c0d66316cd5b802ddcaddf1015d4ceca1aa3b2c5 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -25,139 +25,44 @@ #include "tsync.h" #include "syncInt.h" -static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex) { - char name[TSDB_FILENAME_LEN * 2] = {0}; - char fname[TSDB_FILENAME_LEN * 3] = {0}; - uint32_t magic; - uint64_t fversion; - int64_t size; - uint32_t index = sindex; +static int32_t syncRecvFileVersion(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; - if (sindex < 0 || eindex < sindex) return; - - sDebug("%s, extra files will be removed between sindex:%d and eindex:%d", pPeer->id, sindex, eindex); - - while (1) { - name[0] = 0; - magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion); - if (magic == 0) break; - - snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name); - (void)remove(fname); - sInfo("%s, %s is removed for its extra", pPeer->id, fname); + SFileVersion fileVersion; + memset(&fileVersion, 0, sizeof(SFileVersion)); + int32_t ret = taosReadMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion)); + if (ret != sizeof(SFileVersion)) { + sError("%s, failed to read fver since %s", pPeer->id, strerror(errno)); + return -1; + } - index++; - if (index > eindex) break; + SFileAck fileVersionAck; + memset(&fileVersionAck, 0, sizeof(SFileAck)); + syncBuildFileAck(&fileVersionAck, pNode->vgId); + ret = taosWriteMsg(pPeer->syncFd, &fileVersionAck, sizeof(SFileAck)); + if (ret != sizeof(SFileAck)) { + sError("%s, failed to write fver ack since %s", pPeer->id, strerror(errno)); + return -1; } + + *fversion = htobe64(fileVersion.fversion); + return 0; } static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */ - SFileInfo sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */ - SFileAck fileAck; memset(&fileAck, 0, sizeof(SFileAck)); - int32_t code = -1; - char name[TSDB_FILENAME_LEN * 2] = {0}; - uint32_t pindex = 0; // index in last restore - bool fileChanged = false; - - *fversion = 0; - sinfo.index = -1; - while (1) { - // read file info - minfo.index = -1; - int32_t ret = taosReadMsg(pPeer->syncFd, &minfo, sizeof(SFileInfo)); - if (ret != sizeof(SFileInfo) || minfo.index == -1) { - sError("%s, failed to read fileinfo while restore file since %s", pPeer->id, strerror(errno)); - break; - } - - assert(ret == sizeof(SFileInfo)); - ret = syncCheckHead((SSyncHead *)(&minfo)); - if (ret != 0) { - sError("%s, failed to check fileinfo while restore file since %s", pPeer->id, strerror(ret)); - break; - } - - // if no more file from master, break; - if (minfo.name[0] == 0 || minfo.magic == 0) { - sDebug("%s, no more files to restore", pPeer->id); - - // remove extra files after the current index - if (sinfo.index != -1) syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX); - code = 0; - break; - } - - sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, - minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic); - - // remove extra files on slave between the current and last index - syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1); - pindex = minfo.index; - - // check the file info - sinfo = minfo; - sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); - sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, sinfo.name, - sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic); - - // if file not there or magic is not the same, file shall be synced - memset(&fileAck, 0, sizeof(SFileAck)); - syncBuildFileAck(&fileAck, pNode->vgId); - fileAck.sync = (sinfo.magic != minfo.magic || sinfo.size != minfo.size || sinfo.name[0] == 0) ? 1 : 0; - - // send file ack - ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck)); - if (ret != sizeof(SFileAck)) { - sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno)); - break; - } - // if sync is not required, continue - if (fileAck.sync == 0) { - sDebug("%s, %s is the same", pPeer->id, minfo.name); - continue; - } else { - sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); - } - - // if sync is required, open file, receive from master, and write to file - // get the full path to file - minfo.name[sizeof(minfo.name) - 1] = 0; - snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name); - - int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); - if (dfd < 0) { - sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); - break; - } - - ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size); - fsync(dfd); - close(dfd); - if (ret < 0) { - sError("%s, failed to copy file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); - break; - } - - fileChanged = true; - sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); - } - - if (code == 0 && fileChanged) { - // data file is changed, code shall be set to 1 - *fversion = minfo.fversion; - code = 1; - sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion); + if (pNode->recvFileFp && (*pNode->recvFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) { + sError("%s, failed to restore file", pPeer->id); + return -1; } - if (code < 0) { - sError("%s, failed to restore %s since %s", pPeer->id, name, strerror(errno)); + if (syncRecvFileVersion(pPeer, fversion) < 0) { + return -1; } - return code; + sInfo("%s, all files are restored, fver:%" PRIu64, pPeer->id, *fversion); + return 0; } static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) { @@ -195,7 +100,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) { } lastVer = pHead->version; - ret = (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL); + ret = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL); if (ret != 0) { sError("%s, failed to restore record since %s, hver:%" PRIu64, pPeer->id, tstrerror(ret), pHead->version); break; @@ -215,7 +120,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)offset; - (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); + (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); offset += pHead->len + sizeof(SWalHead); return offset; @@ -226,6 +131,11 @@ static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) { SRecvBuffer *pRecv = pNode->pRecv; int32_t forwards = 0; + if (pRecv == NULL) { + sError("%s, recv buffer is null, restart connect", pPeer->id); + return -1; + } + sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards); char *offset = pRecv->buffer; @@ -274,6 +184,7 @@ int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { static void syncCloseRecvBuffer(SSyncNode *pNode) { if (pNode->pRecv) { + sDebug("vgId:%d, recv buffer:%p is freed", pNode->vgId, pNode->pRecv); tfree(pNode->pRecv->buffer); } @@ -298,6 +209,7 @@ static int32_t syncOpenRecvBuffer(SSyncNode *pNode) { pNode->pRecv = pRecv; + sDebug("vgId:%d, recv buffer:%p is created", pNode->vgId, pNode->pRecv); return 0; } @@ -315,20 +227,16 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { sDebug("%s, send sync rsp to peer, tranId:%u", pPeer->id, rsp.tranId); sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + (*pNode->startSyncFileFp)(pNode->vgId); + int32_t code = syncRestoreFile(pPeer, &fversion); if (code < 0) { - sError("%s, failed to restore file", pPeer->id); + (*pNode->stopSyncFileFp)(pNode->vgId, fversion); + sError("%s, failed to restore files", pPeer->id); return -1; } - // if code > 0, data file is changed, notify app, and pass the version - if (code > 0 && pNode->notifyFileSynced) { - if ((*pNode->notifyFileSynced)(pNode->vgId, fversion) < 0) { - sError("%s, app not in ready state", pPeer->id); - return -1; - } - } - + (*pNode->stopSyncFileFp)(pNode->vgId, fversion); nodeVersion = fversion; sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); @@ -365,10 +273,11 @@ void *syncRestoreData(void *param) { SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); - __sync_fetch_and_add(&tsSyncNum, 1); + atomic_add_fetch_32(&tsSyncNum, 1); sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING); + nodeRole = TAOS_SYNC_ROLE_SYNCING; + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); if (syncOpenRecvBuffer(pNode) < 0) { sError("%s, failed to allocate recv buffer, restart connection", pPeer->id); @@ -385,14 +294,14 @@ void *syncRestoreData(void *param) { } } - (*pNode->notifyRole)(pNode->vgId, nodeRole); + (*pNode->notifyRoleFp)(pNode->vgId, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); syncCloseRecvBuffer(pNode); - __sync_fetch_and_sub(&tsSyncNum, 1); + atomic_sub_fetch_32(&tsSyncNum, 1); // The ref is obtained in both the create thread and the current thread, so it is released twice syncReleasePeer(pPeer); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index cb2379583f853be70bc2e735c81f711a18dc9b52..505ba68c41f2c2373dba3322b6ed31ba0ac853f1 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -14,7 +14,6 @@ */ #define _DEFAULT_SOURCE -#include #include "os.h" #include "taoserror.h" #include "tlog.h" @@ -28,9 +27,9 @@ static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; - int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); if (code != 0) { - sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); + sInfo("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); return -1; } @@ -40,14 +39,14 @@ static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) { static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; - int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); if (code != 0) { - sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); + sInfo("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); return true; } if (wver != pPeer->lastWalVer) { - sDebug("%s, wal is modified while retrieve, wver:%" PRIu64 ", last:%" PRIu64, pPeer->id, wver, pPeer->lastWalVer); + sInfo("%s, wal is modified while retrieve, wver:%" PRIu64 ", last:%" PRIu64, pPeer->id, wver, pPeer->lastWalVer); return true; } @@ -56,9 +55,9 @@ static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) { static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; - int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); if (code != 0) { - sDebug("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); + sInfo("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); return -1; } @@ -68,15 +67,15 @@ static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) { static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { uint64_t fver, wver; - int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + int32_t code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver); if (code != 0) { - sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); + sInfo("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); pPeer->fileChanged = 1; return true; } if (fver != pPeer->lastFileVer) { - sDebug("%s, files are modified while retrieve, fver:%" PRIu64 ", last:%" PRIu64, pPeer->id, fver, pPeer->lastFileVer); + sInfo("%s, files are modified while retrieve, fver:%" PRIu64 ", last:%" PRIu64, pPeer->id, fver, pPeer->lastFileVer); pPeer->fileChanged = 1; return true; } @@ -85,104 +84,54 @@ static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { return false; } -static int32_t syncRetrieveFile(SSyncPeer *pPeer) { +static int32_t syncSendFileVersion(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo)); - SFileAck fileAck; memset(&fileAck, 0, sizeof(SFileAck)); - int32_t code = -1; - char name[TSDB_FILENAME_LEN * 2] = {0}; - if (syncGetFileVersion(pNode, pPeer) < 0) { - pPeer->fileChanged = 1; + SFileVersion fileVersion; + memset(&fileVersion, 0, sizeof(SFileVersion)); + syncBuildFileVersion(&fileVersion, pNode->vgId); + + uint64_t fver = pPeer->lastFileVer; + fileVersion.fversion = htobe64(fver); + int32_t ret = taosWriteMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion)); + if (ret != sizeof(SFileVersion)) { + sError("%s, failed to write fver:%" PRIu64 " since %s", pPeer->id, fver, strerror(errno)); return -1; } - while (1) { - // retrieve file info - fileInfo.name[0] = 0; - fileInfo.size = 0; - fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, - &fileInfo.size, &fileInfo.fversion); - syncBuildFileInfo(&fileInfo, pNode->vgId); - sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, fileInfo.name, - fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic); - - // send the file info - int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo)); - if (ret != sizeof(SFileInfo)) { - code = -1; - sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } - - // if no file anymore, break - if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { - code = 0; - sDebug("%s, no more files to sync", pPeer->id); - break; - } - - // wait for the ack from peer - ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck)); - if (ret != sizeof(SFileAck)) { - code = -1; - sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } - - ret = syncCheckHead((SSyncHead*)(&fileAck)); - if (ret != 0) { - code = -1; - sError("%s, failed to check file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(ret)); - break; - } - - // set the peer sync version - pPeer->sversion = fileInfo.fversion; - - // if sync is not required, continue - if (fileAck.sync == 0) { - fileInfo.index++; - sDebug("%s, %s is the same, fver:%" PRIu64, pPeer->id, fileInfo.name, fileInfo.fversion); - continue; - } else { - sDebug("%s, %s will be sent, fver:%" PRIu64, pPeer->id, fileInfo.name, fileInfo.fversion); - } + SFileAck fileAck; + memset(&fileAck, 0, sizeof(SFileAck)); + ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck)); + if (ret != sizeof(SFileAck)) { + sError("%s, failed to read fver ack since %s", pPeer->id, strerror(errno)); + return -1; + } - // get the full path to file - snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); + // set the peer sync version + pPeer->sversion = fver; - // send the file to peer - int32_t sfd = open(name, O_RDONLY); - if (sfd < 0) { - code = -1; - sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } + return 0; +} - ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); - close(sfd); - if (ret < 0) { - code = -1; - sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } +static int32_t syncRetrieveFile(SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; - sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); - fileInfo.index++; + if (syncGetFileVersion(pNode, pPeer) < 0) { + pPeer->fileChanged = 1; + return -1; + } - // check if processed files are modified - if (syncAreFilesModified(pNode, pPeer)) { - code = -1; - break; - } + if (pNode->sendFileFp && (*pNode->sendFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) { + sError("%s, failed to retrieve file", pPeer->id); + return -1; } - if (code != TSDB_CODE_SUCCESS) { - sError("%s, failed to retrieve file, code:0x%x", pPeer->id, code); + if (syncSendFileVersion(pPeer) < 0) { + return -1; } - return code; + sInfo("%s, all files are retrieved", pPeer->id); + return 0; } // if only a partial record is read out, upper layer will reload the file to get a complete record @@ -194,13 +143,13 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { } if (ret == 0) { - sDebug("sfd:%d, read to the end of file, ret:%d", sfd, ret); + sInfo("sfd:%d, read to the end of file, ret:%d", sfd, ret); return 0; } if (ret != sizeof(SWalHead)) { // file is not at end yet, it shall be reloaded - sDebug("sfd:%d, a partial wal head is read out, ret:%d", sfd, ret); + sInfo("sfd:%d, a partial wal head is read out, ret:%d", sfd, ret); return 0; } @@ -214,31 +163,31 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { if (ret != pHead->len) { // file is not at end yet, it shall be reloaded - sDebug("sfd:%d, a partial wal conetnt is read out, ret:%d", sfd, ret); + sInfo("sfd:%d, a partial wal conetnt is read out, ret:%d", sfd, ret); return 0; } return sizeof(SWalHead) + pHead->len; } -static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) { - int32_t sfd = open(name, O_RDONLY); +static int64_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) { + int32_t sfd = open(name, O_RDONLY | O_BINARY); if (sfd < 0) { sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno)); return -1; } - int32_t code = taosLSeek(sfd, offset, SEEK_SET); + int64_t code = taosLSeek(sfd, offset, SEEK_SET); if (code < 0) { sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno)); close(sfd); return -1; } - sDebug("%s, retrieve last wal:%s, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, name, offset, fversion); + sInfo("%s, retrieve last wal:%s, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, name, offset, fversion); SWalHead *pHead = malloc(SYNC_MAX_SIZE); - int32_t bytes = 0; + int64_t bytes = 0; while (1) { code = syncReadOneWalRecord(sfd, pHead); @@ -249,13 +198,13 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi if (code == 0) { code = bytes; - sDebug("%s, read to the end of wal, bytes:%d", pPeer->id, bytes); + sInfo("%s, read to the end of wal, bytes:%" PRId64, pPeer->id, bytes); break; } - sDebug("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); + sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); - int32_t wsize = code; + int32_t wsize = (int32_t)code; int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); if (ret != wsize) { code = -1; @@ -268,7 +217,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi if (pHead->version >= fversion && fversion > 0) { code = 0; - sDebug("%s, retrieve wal finished, hver:%" PRIu64 " fver:%" PRIu64, pPeer->id, pHead->version, fversion); + sInfo("%s, retrieve wal finished, hver:%" PRIu64 " fver:%" PRIu64, pPeer->id, pHead->version, fversion); break; } } @@ -279,7 +228,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi return code; } -static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { +static int64_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { SSyncNode *pNode = pPeer->pSyncNode; int32_t once = 0; // last WAL has once ever been processed int64_t offset = 0; @@ -288,15 +237,15 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) // get full path to wal file snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); - sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname); + sInfo("%s, start to retrieve last wal:%s", pPeer->id, fname); while (1) { if (syncAreFilesModified(pNode, pPeer)) return -1; if (syncGetWalVersion(pNode, pPeer) < 0) return -1; - int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset); + int64_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset); if (bytes < 0) { - sDebug("%s, failed to retrieve last wal", pPeer->id); + sInfo("%s, failed to retrieve last wal, bytes:%" PRId64, pPeer->id, bytes); return bytes; } @@ -308,13 +257,13 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) if (fversion == 0) { pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt fversion = nodeVersion; // must read data to fversion - sDebug("%s, set sstatus:%s and fver:%" PRIu64, pPeer->id, syncStatus[pPeer->sstatus], fversion); + sInfo("%s, set sstatus:%s and fver:%" PRIu64, pPeer->id, syncStatus[pPeer->sstatus], fversion); } } // if all data up to fversion is read out, it is over if (pPeer->sversion >= fversion && fversion > 0) { - sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d sver:%" PRIu64, pPeer->id, fversion, bytes, + sInfo("%s, data up to fver:%" PRIu64 " has been read out, bytes:%" PRId64 " sver:%" PRIu64, pPeer->id, fversion, bytes, pPeer->sversion); return 0; } @@ -322,44 +271,45 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) // if all data are read out, and no update if (bytes == 0 && !walModified) { // wal not closed, it means some data not flushed to disk, wait for a while - usleep(10000); + taosMsleep(10); } // if bytes > 0, file is updated, or fversion is not reached but file still open, read again once = 1; offset += bytes; - sDebug("%s, continue retrieve last wal, bytes:%d offset:%" PRId64 " sver:%" PRIu64 " fver:%" PRIu64, pPeer->id, + sInfo("%s, continue retrieve last wal, bytes:%" PRId64 " offset:%" PRId64 " sver:%" PRIu64 " fver:%" PRIu64, pPeer->id, bytes, offset, pPeer->sversion, fversion); } return -1; } -static int32_t syncRetrieveWal(SSyncPeer *pPeer) { +static int64_t syncRetrieveWal(SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; char fname[TSDB_FILENAME_LEN * 3]; char wname[TSDB_FILENAME_LEN * 2]; int32_t size; - int32_t code = -1; + int64_t code = -1; int64_t index = 0; while (1) { // retrieve wal info wname[0] = 0; - code = (*pNode->getWalInfo)(pNode->vgId, wname, &index); + code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &index); if (code < 0) { - sError("%s, failed to get wal info since:%s, code:0x%x", pPeer->id, strerror(errno), code); + sError("%s, failed to get wal info since:%s, code:0x%" PRIx64, pPeer->id, strerror(errno), code); break; } if (wname[0] == 0) { // no wal file code = 0; - sDebug("%s, no wal file anymore", pPeer->id); + sInfo("%s, no wal file anymore", pPeer->id); break; } if (code == 0) { // last wal code = syncProcessLastWal(pPeer, wname, index); + sInfo("%s, last wal processed, code:%" PRId64, pPeer->id, code); break; } @@ -370,24 +320,24 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { struct stat fstat; if (stat(fname, &fstat) < 0) { code = -1; - sDebug("%s, failed to stat wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + sInfo("%s, failed to stat wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); break; } size = fstat.st_size; - sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); + sInfo("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); - int32_t sfd = open(fname, O_RDONLY); + int32_t sfd = open(fname, O_RDONLY | O_BINARY); if (sfd < 0) { code = -1; - sError("%s, failed to open wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + sError("%s, failed to open wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); break; } code = taosSendFile(pPeer->syncFd, sfd, NULL, size); close(sfd); if (code < 0) { - sError("%s, failed to send wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + sError("%s, failed to send wal:%s for retrieve since %s, code:0x%" PRIx64, pPeer->id, fname, strerror(errno), code); break; } @@ -408,7 +358,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { code = -1; } } else { - sError("%s, failed to send wal since %s, code:0x%x", pPeer->id, strerror(errno), code); + sError("%s, failed to send wal since %s, code:0x%" PRIx64, pPeer->id, strerror(errno), code); } return code; @@ -424,7 +374,7 @@ static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) { sError("%s, failed to send sync-data msg since %s, tranId:%u", pPeer->id, strerror(errno), msg.tranId); return -1; } - sDebug("%s, send sync-data msg to peer, tranId:%u", pPeer->id, msg.tranId); + sInfo("%s, send sync-data msg to peer, tranId:%u", pPeer->id, msg.tranId); SSyncRsp rsp; if (taosReadMsg(pPeer->syncFd, &rsp, sizeof(SSyncRsp)) != sizeof(SSyncRsp)) { @@ -432,7 +382,7 @@ static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) { return -1; } - sDebug("%s, recv sync-data rsp from peer, tranId:%u rsp-tranId:%u", pPeer->id, msg.tranId, rsp.tranId); + sInfo("%s, recv sync-data rsp from peer, tranId:%u rsp-tranId:%u", pPeer->id, msg.tranId, rsp.tranId); return 0; } @@ -455,9 +405,9 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { if (pPeer->sversion == 0) pPeer->sversion = 1; sInfo("%s, start to retrieve wals", pPeer->id); - int32_t code = syncRetrieveWal(pPeer); - if (code != 0) { - sError("%s, failed to retrieve wals, code:0x%x", pPeer->id, code); + int64_t code = syncRetrieveWal(pPeer); + if (code < 0) { + sError("%s, failed to retrieve wals, code:0x%" PRIx64, pPeer->id, code); return -1; } @@ -478,7 +428,7 @@ void *syncRetrieveData(void *param) { sInfo("%s, start to retrieve data, sstatus:%s, numOfRetrieves:%d", pPeer->id, syncStatus[pPeer->sstatus], pPeer->numOfRetrieves); - if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves); + if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves); pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (pPeer->syncFd < 0) { @@ -498,13 +448,13 @@ void *syncRetrieveData(void *param) { pPeer->numOfRetrieves++; } else { pPeer->numOfRetrieves = 0; - // if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0); + // if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); } - if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0); + if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0); pPeer->fileChanged = 0; - taosClose(pPeer->syncFd); + taosCloseSocket(pPeer->syncFd); // The ref is obtained in both the create thread and the current thread, so it is released twice sInfo("%s, sync retrieve data over, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); diff --git a/src/sync/src/syncTcp.c b/src/sync/src/syncTcp.c index 474466673798ba027518a4bf5c3375492ce9fb60..3ad9e9bba01eb861c48fa121b154af3e7838dda0 100644 --- a/src/sync/src/syncTcp.c +++ b/src/sync/src/syncTcp.c @@ -27,7 +27,7 @@ typedef struct SThreadObj { pthread_t thread; bool stop; - int32_t pollFd; + SOCKET pollFd; int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -37,13 +37,14 @@ typedef struct SPoolObj { SThreadObj **pThread; pthread_t thread; int32_t nextId; - int32_t acceptFd; // FD for accept new connection + SOCKET acceptFd; // FD for accept new connection + int8_t stop; } SPoolObj; typedef struct { SThreadObj *pThread; int64_t handleId; - int32_t fd; + SOCKET fd; int32_t closedByApp; } SConnObj; @@ -82,7 +83,7 @@ void *syncOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pPool->thread), &thattr, (void *)syncAcceptPeerTcpConnection, pPool) != 0) { sError("failed to create accept thread for TCP server since %s", strerror(errno)); - close(pPool->acceptFd); + taosCloseSocket(pPool->acceptFd); tfree(pPool->pThread); tfree(pPool); return NULL; @@ -98,7 +99,19 @@ void syncCloseTcpThreadPool(void *param) { SPoolObj * pPool = param; SThreadObj *pThread; + pPool->stop = 1; + +#ifdef WINDOWS + closesocket(pPool->acceptFd); +#elif defined(__APPLE__) + if (pPool->acceptFd!=-1) { + close(pPool->acceptFd); + pPool->acceptFd = -1; + } +#else shutdown(pPool->acceptFd, SHUT_RD); +#endif + pthread_join(pPool->thread, NULL); for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) { @@ -112,7 +125,7 @@ void syncCloseTcpThreadPool(void *param) { tfree(pPool); } -void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) { +void *syncAllocateTcpConn(void *param, int64_t rid, SOCKET connFd) { struct epoll_event event; SPoolObj *pPool = param; @@ -169,7 +182,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); - taosClose(pConn->fd); + taosCloseSocket(pConn->fd); tfree(pConn); } @@ -233,7 +246,7 @@ static void *syncProcessTcpData(void *param) { sDebug("%p TCP epoll thread exits", pThread); - close(pThread->pollFd); + EpollClose(pThread->pollFd); tfree(pThread); tfree(buffer); return NULL; @@ -248,7 +261,12 @@ static void *syncAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + SOCKET connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + if (pPool->stop) { + sDebug("%p TCP server accept is stopped", pPool); + break; + } + if (connFd < 0) { if (errno == EINVAL) { sDebug("%p TCP server accept is exiting...", pPool); @@ -264,7 +282,7 @@ static void *syncAcceptPeerTcpConnection(void *argv) { (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); } - taosClose(pPool->acceptFd); + taosCloseSocket(pPool->acceptFd); return NULL; } @@ -277,7 +295,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { if (pThread == NULL) return NULL; pThread->pPool = pPool; - pThread->pollFd = epoll_create(10); // size does not matter + pThread->pollFd = (EpollFd)epoll_create(10); // size does not matter if (pThread->pollFd < 0) { tfree(pThread); return NULL; @@ -290,7 +308,7 @@ static SThreadObj *syncGetTcpThread(SPoolObj *pPool) { pthread_attr_destroy(&thattr); if (ret != 0) { - close(pThread->pollFd); + EpollClose(pThread->pollFd); tfree(pThread); return NULL; } diff --git a/src/sync/test/CMakeLists.txt b/src/sync/test/CMakeLists.txt index 256e87580d9372f0ade7200591303211f0140952..f2b05ab2263c0d80bc870981f86933922de639e4 100644 --- a/src/sync/test/CMakeLists.txt +++ b/src/sync/test/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_LINUX) @@ -13,4 +13,3 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(syncServer sync trpc common) ENDIF () - diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 161105d86c9a45fb1b8827f01701a43a3e08000c..eeaa6a08c2e47d103b62d6023dde74341585f65f 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -296,11 +296,10 @@ void initSync() { pCfg->replica = 1; pCfg->quorum = 1; syncInfo.vgId = 1; - syncInfo.getFileInfo = getFileInfo; - syncInfo.getWalInfo = getWalInfo; - syncInfo.writeToCache = writeToCache; + syncInfo.getWalInfoFp = getWalInfo; + syncInfo.writeToCacheFp = writeToCache; syncInfo.confirmForward = confirmForward; - syncInfo.notifyRole = notifyRole; + syncInfo.notifyRoleFp = notifyRole; pCfg->nodeInfo[0].nodeId = 1; pCfg->nodeInfo[0].nodePort = 7010; diff --git a/src/tfs/CMakeLists.txt b/src/tfs/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b435c84366fb47bd137b1c13bc98eab625bbcc66 --- /dev/null +++ b/src/tfs/CMakeLists.txt @@ -0,0 +1,12 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +INCLUDE_DIRECTORIES(inc) +AUX_SOURCE_DIRECTORY(src SRC) +ADD_LIBRARY(tfs ${SRC}) +TARGET_LINK_LIBRARIES(tfs tutil) + +IF (TD_LINUX) + # Someone has no gtest directory, so comment it + # ADD_SUBDIRECTORY(tests) +ENDIF () diff --git a/src/tfs/inc/tfsint.h b/src/tfs/inc/tfsint.h new file mode 100644 index 0000000000000000000000000000000000000000..619ef6df73444934a949955af41c85f359660116 --- /dev/null +++ b/src/tfs/inc/tfsint.h @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TD_TFSINT_H +#define TD_TFSINT_H + +#include "tlog.h" +#include "tglobal.h" +#include "tfs.h" +#include "tcoding.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern int fsDebugFlag; + +// For debug purpose +#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }} +#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }} +#define fWarn(...) { if (fsDebugFlag & DEBUG_WARN) { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }} +#define fInfo(...) { if (fsDebugFlag & DEBUG_INFO) { taosPrintLog("TFS ", 255, __VA_ARGS__); }} +#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }} +#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }} + +// Global Definitions +#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 + +// tdisk.c ====================================================== +typedef struct { + int64_t size; + int64_t used; + int64_t free; +} SDiskMeta; + +typedef struct SDisk { + int level; + int id; + char dir[TSDB_FILENAME_LEN]; + SDiskMeta dmeta; +} SDisk; + +#define DISK_LEVEL(pd) ((pd)->level) +#define DISK_ID(pd) ((pd)->id) +#define DISK_DIR(pd) ((pd)->dir) +#define DISK_META(pd) ((pd)->dmeta) +#define DISK_SIZE(pd) ((pd)->dmeta.size) +#define DISK_USED_SIZE(pd) ((pd)->dmeta.used) +#define DISK_FREE_SIZE(pd) ((pd)->dmeta.free) + +SDisk *tfsNewDisk(int level, int id, const char *dir); +SDisk *tfsFreeDisk(SDisk *pDisk); +int tfsUpdateDiskInfo(SDisk *pDisk); + +// ttier.c ====================================================== +typedef struct { + int64_t size; + int64_t used; + int64_t free; + int16_t nAvailDisks; // # of Available disks +} STierMeta; +typedef struct STier { + pthread_spinlock_t lock; + int level; + int16_t ndisk; // # of disks mounted to this tier + int16_t nextid; // next disk id to allocate + STierMeta tmeta; + SDisk * disks[TSDB_MAX_DISKS_PER_TIER]; +} STier; + +#define TIER_LEVEL(pt) ((pt)->level) +#define TIER_NDISKS(pt) ((pt)->ndisk) +#define TIER_SIZE(pt) ((pt)->tmeta.size) +#define TIER_FREE_SIZE(pt) ((pt)->tmeta.free) +#define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks) +#define DISK_AT_TIER(pt, id) ((pt)->disks[id]) + +int tfsInitTier(STier *pTier, int level); +void tfsDestroyTier(STier *pTier); +SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg); +void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta); +int tfsAllocDiskOnTier(STier *pTier); +void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta); +void tfsPosNextId(STier *pTier); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/tfs/src/tdisk.c b/src/tfs/src/tdisk.c new file mode 100644 index 0000000000000000000000000000000000000000..22601e48c3607fca53c91ea00b17551c791302bd --- /dev/null +++ b/src/tfs/src/tdisk.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" + +#include "taoserror.h" +#include "tfsint.h" + +// PROTECTED ==================================== +SDisk *tfsNewDisk(int level, int id, const char *dir) { + SDisk *pDisk = (SDisk *)calloc(1, sizeof(*pDisk)); + if (pDisk == NULL) { + terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + return NULL; + } + + pDisk->level = level; + pDisk->id = id; + tstrncpy(pDisk->dir, dir, TSDB_FILENAME_LEN); + + return pDisk; +} + +SDisk *tfsFreeDisk(SDisk *pDisk) { + if (pDisk) { + free(pDisk); + } + return NULL; +} + +int tfsUpdateDiskInfo(SDisk *pDisk) { + ASSERT(pDisk != NULL); + + SysDiskSize diskSize = {0}; + + int code = taosGetDiskSize(pDisk->dir, &diskSize); + if (code != 0) { + fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + } + + pDisk->dmeta.size = diskSize.tsize; + pDisk->dmeta.used = diskSize.used; + pDisk->dmeta.free = diskSize.avail; + + return code; +} diff --git a/src/tfs/src/tfs.c b/src/tfs/src/tfs.c new file mode 100644 index 0000000000000000000000000000000000000000..f78535b8ed5bd17cb232fbcbc73f9831db099547 --- /dev/null +++ b/src/tfs/src/tfs.c @@ -0,0 +1,602 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" + +#include "hash.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tfs.h" +#include "tfsint.h" + +#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32) + +typedef struct { + pthread_spinlock_t lock; + SFSMeta meta; + int nlevel; + STier tiers[TSDB_MAX_TIERS]; + SHashObj * map; // name to did map +} SFS; + +typedef struct { + SDisk *pDisk; +} SDiskIter; + +#define TFS_META() (pfs->meta) +#define TFS_NLEVEL() (pfs->nlevel) +#define TFS_TIERS() (pfs->tiers) +#define TFS_TIER_AT(level) (TFS_TIERS() + (level)) +#define TFS_DISK_AT(level, id) DISK_AT_TIER(TFS_TIER_AT(level), id) +#define TFS_PRIMARY_DISK() TFS_DISK_AT(TFS_PRIMARY_LEVEL, TFS_PRIMARY_ID) +#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL())) +#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level)))) +#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id)) + +#define tfsLock() pthread_spin_lock(&(pfs->lock)) +#define tfsUnLock() pthread_spin_unlock(&(pfs->lock)) + +static SFS tfs = {0}; +static SFS *pfs = &tfs; + +// STATIC DECLARATION +static int tfsMount(SDiskCfg *pCfg); +static int tfsCheck(); +static int tfsCheckAndFormatCfg(SDiskCfg *pCfg); +static int tfsFormatDir(char *idir, char *odir); +static SDisk *tfsGetDiskByID(SDiskID did); +static SDisk *tfsGetDiskByName(const char *dir); +static int tfsOpendirImpl(TDIR *tdir); +static void tfsInitDiskIter(SDiskIter *pIter); +static SDisk *tfsNextDisk(SDiskIter *pIter); + +// FS APIs ==================================== +int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { + ASSERT(ndisk > 0); + + for (int level = 0; level < TSDB_MAX_TIERS; level++) { + if (tfsInitTier(TFS_TIER_AT(level), level) < 0) { + while (true) { + level--; + if (level < 0) break; + + tfsDestroyTier(TFS_TIER_AT(level)); + } + + return -1; + } + } + + pthread_spin_init(&(pfs->lock), 0); + + pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pfs->map == NULL) { + terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + tfsDestroy(); + return -1; + } + + for (int idisk = 0; idisk < ndisk; idisk++) { + if (tfsMount(pDiskCfg + idisk) < 0) { + tfsDestroy(); + return -1; + } + } + + if (tfsCheck() < 0) { + tfsDestroy(); + return -1; + } + + tfsUpdateInfo(NULL); + for (int level = 0; level < TFS_NLEVEL(); level++) { + tfsPosNextId(TFS_TIER_AT(level)); + } + + return 0; +} + +void tfsDestroy() { + taosHashCleanup(pfs->map); + pfs->map = NULL; + + pthread_spin_destroy(&(pfs->lock)); + for (int level = 0; level < TFS_NLEVEL(); level++) { + tfsDestroyTier(TFS_TIER_AT(level)); + } +} + +void tfsUpdateInfo(SFSMeta *pFSMeta) { + SFSMeta fsMeta; + STierMeta tierMeta; + + if (pFSMeta == NULL) { + pFSMeta = &fsMeta; + } + + memset(pFSMeta, 0, sizeof(*pFSMeta)); + + for (int level = 0; level < TFS_NLEVEL(); level++) { + STier *pTier = TFS_TIER_AT(level); + tfsUpdateTierInfo(pTier, &tierMeta); + pFSMeta->tsize += tierMeta.size; + pFSMeta->avail += tierMeta.free; + pFSMeta->used += tierMeta.used; + } + + tfsLock(); + pfs->meta = *pFSMeta; + tfsUnLock(); +} + +void tfsGetMeta(SFSMeta *pMeta) { + ASSERT(pMeta); + + tfsLock(); + *pMeta = pfs->meta; + tfsUnLock(); +} + +/* Allocate an existing available tier level + */ +void tfsAllocDisk(int expLevel, int *level, int *id) { + ASSERT(expLevel >= 0); + + *level = expLevel; + *id = TFS_UNDECIDED_ID; + + if (*level >= TFS_NLEVEL()) { + *level = TFS_NLEVEL() - 1; + } + + while (*level >= 0) { + *id = tfsAllocDiskOnTier(TFS_TIER_AT(*level)); + if (*id == TFS_UNDECIDED_ID) { + (*level)--; + continue; + } + + return; + } + + *level = TFS_UNDECIDED_LEVEL; + *id = TFS_UNDECIDED_ID; +} + +const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); } +const char *TFS_DISK_PATH(int level, int id) { return DISK_DIR(TFS_DISK_AT(level, id)); } + +// TFILE APIs ==================================== +void tfsInitFile(TFILE *pf, int level, int id, const char *bname) { + ASSERT(TFS_IS_VALID_DISK(level, id)); + + SDisk *pDisk = TFS_DISK_AT(level, id); + + pf->level = level; + pf->id = id; + tstrncpy(pf->rname, bname, TSDB_FILENAME_LEN); + + char tmpName[TMPNAME_LEN] = {0}; + snprintf(tmpName, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), bname); + tstrncpy(pf->aname, tmpName, TSDB_FILENAME_LEN); +} + +bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) { + ASSERT(pf1 != NULL || pf2 != NULL); + if (pf1 == NULL || pf2 == NULL) return false; + if (pf1->level != pf2->level) return false; + if (pf1->id != pf2->id) return false; + if (strncmp(pf1->rname, pf2->rname, TSDB_FILENAME_LEN) != 0) return false; + return true; +} + +int tfsEncodeFile(void **buf, TFILE *pf) { + int tlen = 0; + + tlen += taosEncodeVariantI32(buf, pf->level); + tlen += taosEncodeVariantI32(buf, pf->id); + tlen += taosEncodeString(buf, pf->rname); + + return tlen; +} + +void *tfsDecodeFile(void *buf, TFILE *pf) { + int32_t level, id; + char * rname; + + buf = taosDecodeVariantI32(buf, &(level)); + buf = taosDecodeVariantI32(buf, &(id)); + buf = taosDecodeString(buf, &rname); + + tfsInitFile(pf, level, id, rname); + + tfree(rname); + return buf; +} + +void tfsbasename(const TFILE *pf, char *dest) { + char tname[TSDB_FILENAME_LEN] = "\0"; + + tstrncpy(tname, pf->aname, TSDB_FILENAME_LEN); + tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN); +} + +void tfsdirname(const TFILE *pf, char *dest) { + char tname[TSDB_FILENAME_LEN] = "\0"; + + tstrncpy(tname, pf->aname, TSDB_FILENAME_LEN); + tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN); +} + +// DIR APIs ==================================== +int tfsMkdirAt(const char *rname, int level, int id) { + SDisk *pDisk = TFS_DISK_AT(level, id); + char aname[TMPNAME_LEN]; + + snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); + if (taosMkDir(aname, 0755) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +int tfsMkdirRecurAt(const char *rname, int level, int id) { + if (tfsMkdirAt(rname, level, id) < 0) { + if (errno == ENOENT) { + // Try to create upper + char *s = strdup(rname); + + if (tfsMkdirRecurAt(dirname(s), level, id) < 0) { + tfree(s); + return -1; + } + tfree(s); + + if (tfsMkdirAt(rname, level, id) < 0) { + return -1; + } + } else { + return -1; + } + } + + return 0; +} + +int tfsMkdir(const char *rname) { + for (int level = 0; level < TFS_NLEVEL(); level++) { + STier *pTier = TFS_TIER_AT(level); + for (int id = 0; id < TIER_NDISKS(pTier); id++) { + if (tfsMkdirAt(rname, level, id) < 0) { + return -1; + } + } + } + + return 0; +} + +int tfsRmdir(const char *rname) { + char aname[TMPNAME_LEN] = "\0"; + + for (int level = 0; level < TFS_NLEVEL(); level++) { + STier *pTier = TFS_TIER_AT(level); + for (int id = 0; id < TIER_NDISKS(pTier); id++) { + SDisk *pDisk = DISK_AT_TIER(pTier, id); + + snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname); + + taosRemoveDir(aname); + } + } + + return 0; +} + +int tfsRename(char *orname, char *nrname) { + char oaname[TMPNAME_LEN] = "\0"; + char naname[TMPNAME_LEN] = "\0"; + + for (int level = 0; level < pfs->nlevel; level++) { + STier *pTier = TFS_TIER_AT(level); + for (int id = 0; id < TIER_NDISKS(pTier); id++) { + SDisk *pDisk = DISK_AT_TIER(pTier, id); + + snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname); + snprintf(naname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), nrname); + + taosRename(oaname, naname); + } + } + + return 0; +} + +struct TDIR { + SDiskIter iter; + int level; + int id; + char dirname[TSDB_FILENAME_LEN]; + TFILE tfile; + DIR * dir; +}; + +TDIR *tfsOpendir(const char *rname) { + TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir)); + if (tdir == NULL) { + terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + return NULL; + } + + tfsInitDiskIter(&(tdir->iter)); + tstrncpy(tdir->dirname, rname, TSDB_FILENAME_LEN); + + if (tfsOpendirImpl(tdir) < 0) { + free(tdir); + return NULL; + } + + return tdir; +} + +const TFILE *tfsReaddir(TDIR *tdir) { + if (tdir == NULL || tdir->dir == NULL) return NULL; + char bname[TMPNAME_LEN * 2] = "\0"; + + while (true) { + struct dirent *dp = NULL; + dp = readdir(tdir->dir); + if (dp != NULL) { + // Skip . and .. + if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; + + snprintf(bname, TMPNAME_LEN * 2, "%s/%s", tdir->dirname, dp->d_name); + tfsInitFile(&(tdir->tfile), tdir->level, tdir->id, bname); + return &(tdir->tfile); + } + + if (tfsOpendirImpl(tdir) < 0) { + return NULL; + } + + if (tdir->dir == NULL) { + terrno = TSDB_CODE_SUCCESS; + return NULL; + } + } +} + +void tfsClosedir(TDIR *tdir) { + if (tdir) { + if (tdir->dir != NULL) { + closedir(tdir->dir); + tdir->dir = NULL; + } + free(tdir); + } +} + +// private +static int tfsMount(SDiskCfg *pCfg) { + SDiskID did; + SDisk * pDisk = NULL; + + if (tfsCheckAndFormatCfg(pCfg) < 0) return -1; + + did.level = pCfg->level; + pDisk = tfsMountDiskToTier(TFS_TIER_AT(did.level), pCfg); + if (pDisk == NULL) { + fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno)); + return -1; + } + did.id = DISK_ID(pDisk); + + taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did)); + if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1; + + return 0; +} + +static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) { + char dirName[TSDB_FILENAME_LEN] = "\0"; + struct stat pstat; + + if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) { + fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (pCfg->primary) { + if (pCfg->level != 0) { + fError("failed to mount %s to FS since disk is primary but level %d not 0", pCfg->dir, pCfg->level); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (TFS_PRIMARY_DISK() != NULL) { + fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir); + terrno = TSDB_CODE_FS_DUP_PRIMARY; + return -1; + } + } + + if (tfsFormatDir(pCfg->dir, dirName) < 0) { + fError("failed to mount %s to FS since invalid dir format", pCfg->dir); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (tfsGetDiskByName(dirName) != NULL) { + fError("failed to mount %s to FS since duplicate mount", pCfg->dir); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (access(dirName, W_OK | R_OK | F_OK) != 0) { + fError("failed to mount %s to FS since no R/W access rights", pCfg->dir); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + if (stat(dirName, &pstat) < 0) { + fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + if (!S_ISDIR(pstat.st_mode)) { + fError("failed to mount %s to FS since not a directory", pCfg->dir); + terrno = TSDB_CODE_FS_INVLD_CFG; + return -1; + } + + strncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN); + + return 0; +} + +static int tfsFormatDir(char *idir, char *odir) { + wordexp_t wep = {0}; + + int code = wordexp(idir, &wep, 0); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + if (realpath(wep.we_wordv[0], odir) == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + wordfree(&wep); + return -1; + } + + wordfree(&wep); + return 0; + +} + +static int tfsCheck() { + if (TFS_PRIMARY_DISK() == NULL) { + fError("no primary disk is set"); + terrno = TSDB_CODE_FS_NO_PRIMARY_DISK; + return -1; + } + + for (int level = 0; level < TFS_NLEVEL(); level++) { + if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) { + fError("no disk at level %d", level); + terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER; + return -1; + } + } + + return 0; +} + +static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); } +static SDisk *tfsGetDiskByName(const char *dir) { + SDiskID did; + SDisk * pDisk = NULL; + void * pr = NULL; + + pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN)); + if (pr == NULL) return NULL; + + did = *(SDiskID *)pr; + pDisk = tfsGetDiskByID(did); + ASSERT(pDisk != NULL); + + return pDisk; +} + +static int tfsOpendirImpl(TDIR *tdir) { + SDisk *pDisk = NULL; + char adir[TMPNAME_LEN * 2] = "\0"; + + if (tdir->dir != NULL) { + closedir(tdir->dir); + tdir->dir = NULL; + } + + while (true) { + pDisk = tfsNextDisk(&(tdir->iter)); + if (pDisk == NULL) return 0; + + tdir->level = DISK_LEVEL(pDisk); + tdir->id = DISK_ID(pDisk); + + snprintf(adir, TMPNAME_LEN * 2, "%s/%s", DISK_DIR(pDisk), tdir->dirname); + tdir->dir = opendir(adir); + if (tdir->dir != NULL) break; + } + + return 0; +} + +static void tfsInitDiskIter(SDiskIter *pIter) { pIter->pDisk = TFS_DISK_AT(0, 0); } + +static SDisk *tfsNextDisk(SDiskIter *pIter) { + SDisk *pDisk = pIter->pDisk; + + if (pDisk == NULL) return NULL; + + int level = DISK_LEVEL(pDisk); + int id = DISK_ID(pDisk); + + id++; + if (id < TIER_NDISKS(TFS_TIER_AT(level))) { + pIter->pDisk = TFS_DISK_AT(level, id); + ASSERT(pIter->pDisk != NULL); + } else { + level++; + id = 0; + if (level < TFS_NLEVEL()) { + pIter->pDisk = TFS_DISK_AT(level, id); + ASSERT(pIter->pDisk != NULL); + } else { + pIter->pDisk = NULL; + } + } + + return pDisk; +} + +// OTHER FUNCTIONS =================================== +void taosGetDisk() { + const double unit = 1024 * 1024 * 1024; + SysDiskSize diskSize; + SFSMeta fsMeta; + + if (tscEmbedded) { + tfsUpdateInfo(&fsMeta); + tsTotalDataDirGB = (float)(fsMeta.tsize / unit); + tsUsedDataDirGB = (float)(fsMeta.used / unit); + tsAvailDataDirGB = (float)(fsMeta.avail / unit); + } + + if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { + tsTotalLogDirGB = (float)(diskSize.tsize / unit); + tsAvailLogDirGB = (float)(diskSize.avail / unit); + } + + if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { + tsTotalTmpDirGB = (float)(diskSize.tsize / unit); + tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); + } +} diff --git a/src/tfs/src/ttier.c b/src/tfs/src/ttier.c new file mode 100644 index 0000000000000000000000000000000000000000..3b19797acf67f36a31c30dccefc0c985512f510b --- /dev/null +++ b/src/tfs/src/ttier.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "os.h" + +#include "taosdef.h" +#include "taoserror.h" +#include "tfsint.h" + +#define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock)) +#define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock)) + +// PROTECTED ========================================== +int tfsInitTier(STier *pTier, int level) { + memset((void *)pTier, 0, sizeof(*pTier)); + + int code = pthread_spin_init(&(pTier->lock), 0); + if (code) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + pTier->level = level; + + return 0; +} + +void tfsDestroyTier(STier *pTier) { + for (int id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) { + DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id)); + } + + pTier->ndisk = 0; + + pthread_spin_destroy(&(pTier->lock)); +} + +SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) { + ASSERT(pTier->level == pCfg->level); + + int id = 0; + SDisk *pDisk; + + if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) { + terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; + return NULL; + } + + if (pTier->level == 0) { + if (DISK_AT_TIER(pTier, 0) != NULL) { + id = pTier->ndisk; + } else { + if (pCfg->primary) { + id = 0; + } else { + id = pTier->ndisk + 1; + } + if (id >= TSDB_MAX_DISKS_PER_TIER) { + terrno = TSDB_CODE_FS_TOO_MANY_MOUNT; + return NULL; + } + } + } else { + id = pTier->ndisk; + } + + pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir); + if (pDisk == NULL) return NULL; + DISK_AT_TIER(pTier, id) = pDisk; + pTier->ndisk++; + + fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id); + + return DISK_AT_TIER(pTier, id); +} + +void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) { + STierMeta tmeta; + + if (pTierMeta == NULL) { + pTierMeta = &tmeta; + } + memset(pTierMeta, 0, sizeof(*pTierMeta)); + + tfsLockTier(pTier); + + for (int id = 0; id < pTier->ndisk; id++) { + if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) < 0) { + continue; + } + pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id)); + pTierMeta->used += DISK_USED_SIZE(DISK_AT_TIER(pTier, id)); + pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id)); + pTierMeta->nAvailDisks++; + } + + pTier->tmeta = *pTierMeta; + + tfsUnLockTier(pTier); +} + +// Round-Robin to allocate disk on a tier +int tfsAllocDiskOnTier(STier *pTier) { + ASSERT(pTier->ndisk > 0); + int id = TFS_UNDECIDED_ID; + SDisk *pDisk; + + tfsLockTier(pTier); + + if (TIER_AVAIL_DISKS(pTier) <= 0) { + tfsUnLockTier(pTier); + return id; + } + + id = pTier->nextid; + while (true) { + pDisk = DISK_AT_TIER(pTier, id); + ASSERT(pDisk != NULL); + + if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) { + id = (id + 1) % pTier->ndisk; + if (id == pTier->nextid) { + tfsUnLockTier(pTier); + return TFS_UNDECIDED_ID; + } else { + continue; + } + } else { + pTier->nextid = (id + 1) % pTier->ndisk; + break; + } + } + + tfsUnLockTier(pTier); + return id; +} + +void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) { + ASSERT(pTierMeta != NULL); + + tfsLockTier(pTier); + *pTierMeta = pTier->tmeta; + tfsUnLockTier(pTier); +} + +void tfsPosNextId(STier *pTier) { + ASSERT(pTier->ndisk > 0); + int nextid = 0; + + for (int id = 1; id < pTier->ndisk; id++) { + SDisk *pLDisk = DISK_AT_TIER(pTier, nextid); + SDisk *pDisk = DISK_AT_TIER(pTier, id); + if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) { + nextid = id; + } + } + + pTier->nextid = nextid; +} diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt index d8bc20ca9971a4bd398e9c5d1d673dd30acc7dc5..31d52aae7d4a809044ab01a7b561801d1ad0c2eb 100644 --- a/src/tsdb/CMakeLists.txt +++ b/src/tsdb/CMakeLists.txt @@ -1,10 +1,10 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tsdb ${SRC}) -TARGET_LINK_LIBRARIES(tsdb common tutil) +TARGET_LINK_LIBRARIES(tsdb tfs common tutil) IF (TD_LINUX) # Someone has no gtest directory, so comment it diff --git a/src/tsdb/inc/tsdbBuffer.h b/src/tsdb/inc/tsdbBuffer.h new file mode 100644 index 0000000000000000000000000000000000000000..414ace00097d95742080a8f173177d5e44497237 --- /dev/null +++ b/src/tsdb/inc/tsdbBuffer.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_BUFFER_H_ +#define _TD_TSDB_BUFFER_H_ + +typedef struct { + int64_t blockId; + int offset; + int remain; + char data[]; +} STsdbBufBlock; + +typedef struct { + pthread_cond_t poolNotEmpty; + int bufBlockSize; + int tBufBlocks; + int nBufBlocks; + int64_t index; + SList* bufBlockList; +} STsdbBufPool; + +#define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold + +STsdbBufPool* tsdbNewBufPool(); +void tsdbFreeBufPool(STsdbBufPool* pBufPool); +int tsdbOpenBufPool(STsdbRepo* pRepo); +void tsdbCloseBufPool(STsdbRepo* pRepo); +SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); + +#endif /* _TD_TSDB_BUFFER_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbCommit.h b/src/tsdb/inc/tsdbCommit.h new file mode 100644 index 0000000000000000000000000000000000000000..5e740081d187466435860a3a6c066412419ab571 --- /dev/null +++ b/src/tsdb/inc/tsdbCommit.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_COMMIT_H_ +#define _TD_TSDB_COMMIT_H_ + +typedef struct { + int minFid; + int midFid; + int maxFid; + TSKEY minKey; +} SRtn; + +typedef struct { + uint64_t uid; + int64_t offset; + int64_t size; +} SKVRecord; + +void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn); +int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord); +void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord); +void *tsdbCommitData(STsdbRepo *pRepo); + +static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) { + if (fid >= pRtn->maxFid) { + return 0; + } else if (fid >= pRtn->midFid) { + return 1; + } else if (fid >= pRtn->minFid) { + return 2; + } else { + return -1; + } +} + +#endif /* _TD_TSDB_COMMIT_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbCommitQueue.h b/src/tsdb/inc/tsdbCommitQueue.h new file mode 100644 index 0000000000000000000000000000000000000000..c2353391f9ad88185097eb52907e890c96021f8c --- /dev/null +++ b/src/tsdb/inc/tsdbCommitQueue.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_COMMIT_QUEUE_H_ +#define _TD_TSDB_COMMIT_QUEUE_H_ + +int tsdbScheduleCommit(STsdbRepo *pRepo); + +#endif /* _TD_TSDB_COMMIT_QUEUE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbFS.h b/src/tsdb/inc/tsdbFS.h new file mode 100644 index 0000000000000000000000000000000000000000..d63aeb14ac6ca6cd6f59654cf74f11d8e33d6ce4 --- /dev/null +++ b/src/tsdb/inc/tsdbFS.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_FS_H_ +#define _TD_TSDB_FS_H_ + +#define TSDB_FS_VERSION 0 + +// ================== CURRENT file header info +typedef struct { + uint32_t version; // Current file system version (relating to code) + uint32_t len; // Encode content length (including checksum) +} SFSHeader; + +// ================== TSDB File System Meta +typedef struct { + uint32_t version; // Commit version from 0 to increase + int64_t totalPoints; // total points + int64_t totalStorage; // Uncompressed total storage +} STsdbFSMeta; + +// ================== +typedef struct { + STsdbFSMeta meta; // FS meta + SMFile* pmf; // meta file pointer + SMFile mf; // meta file + SArray* df; // data file array +} SFSStatus; + +typedef struct { + pthread_rwlock_t lock; + + SFSStatus* cstatus; // current status + SHashObj* metaCache; // meta cache + bool intxn; + SFSStatus* nstatus; // new status +} STsdbFS; + +#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus) +#define FS_NEW_STATUS(pfs) ((pfs)->nstatus) +#define FS_IN_TXN(pfs) (pfs)->intxn +#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version) +#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version) + +typedef struct { + int direction; + uint64_t version; // current FS version + STsdbFS* pfs; + int index; // used to position next fset when version the same + int fid; // used to seek when version is changed + SDFileSet* pSet; +} SFSIter; + +#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC +#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC + +STsdbFS *tsdbNewFS(STsdbCfg *pCfg); +void * tsdbFreeFS(STsdbFS *pfs); +int tsdbOpenFS(STsdbRepo *pRepo); +void tsdbCloseFS(STsdbRepo *pRepo); +void tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd); +int tsdbEndFSTxn(STsdbRepo *pRepo); +int tsdbEndFSTxnWithError(STsdbFS *pfs); +void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta); +void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile); +int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet); + +void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction); +void tsdbFSIterSeek(SFSIter *pIter, int fid); +SDFileSet *tsdbFSIterNext(SFSIter *pIter); +int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta); + +static FORCE_INLINE int tsdbRLockFS(STsdbFS* pFs) { + int code = pthread_rwlock_rdlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int tsdbWLockFS(STsdbFS* pFs) { + int code = pthread_rwlock_wrlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) { + int code = pthread_rwlock_unlock(&(pFs->lock)); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +#endif /* _TD_TSDB_FS_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h new file mode 100644 index 0000000000000000000000000000000000000000..792efcdb2e0cc0245203d91e80369205cab74d56 --- /dev/null +++ b/src/tsdb/inc/tsdbFile.h @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TS_TSDB_FILE_H_ +#define _TS_TSDB_FILE_H_ + +#define TSDB_FILE_HEAD_SIZE 512 +#define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF +#define TSDB_IVLD_FID INT_MIN +#define TSDB_FILE_STATE_OK 0 +#define TSDB_FILE_STATE_BAD 1 + +#define TSDB_FILE_INFO(tf) (&((tf)->info)) +#define TSDB_FILE_F(tf) (&((tf)->f)) +#define TSDB_FILE_FD(tf) ((tf)->fd) +#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf)) +#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) +#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) +#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) +#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf)) +#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf)) +#define TSDB_FILE_FSYNC(tf) fsync(TSDB_FILE_FD(tf)) +#define TSDB_FILE_STATE(tf) ((tf)->state) +#define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) +#define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) +#define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) + +typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T; + +// =============== SMFile +typedef struct { + int64_t size; + int64_t tombSize; + int64_t nRecords; + int64_t nDels; + uint32_t magic; +} SMFInfo; + +typedef struct { + SMFInfo info; + TFILE f; + int fd; + uint8_t state; +} SMFile; + +void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); +void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile); +int tsdbEncodeSMFile(void** buf, SMFile* pMFile); +void* tsdbDecodeSMFile(void* buf, SMFile* pMFile); +int tsdbEncodeSMFileEx(void** buf, SMFile* pMFile); +void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile); +int tsdbApplyMFileChange(SMFile* from, SMFile* to); +int tsdbCreateMFile(SMFile* pMFile, bool updateHeader); +int tsdbUpdateMFileHeader(SMFile* pMFile); +int tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo); +int tsdbScanAndTryFixMFile(STsdbRepo* pRepo); +int tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo); +void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo); + +static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; } + +static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) { + ASSERT(TSDB_FILE_CLOSED(pMFile)); + + pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags); + if (pMFile->fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) { + if (TSDB_FILE_OPENED(pMFile)) { + close(pMFile->fd); + TSDB_FILE_SET_CLOSED(pMFile); + } +} + +static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) { + ASSERT(TSDB_FILE_OPENED(pMFile)); + + int64_t loffset = taosLSeek(TSDB_FILE_FD(pMFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pMFile)); + + int64_t nwrite = taosWrite(pMFile->fd, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) { + pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM)); +} + +static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) { + ASSERT(TSDB_FILE_OPENED(pMFile)); + + int64_t toffset; + + if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pMFile->info.size == toffset); + + if (offset) { + *offset = toffset; + } + + if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) { + return -1; + } + + pMFile->info.size += nbyte; + + return (int)nbyte; +} + +static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); } + +static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pMFile)); + + int64_t nread = taosRead(pMFile->fd, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +// =============== SDFile +typedef struct { + uint32_t magic; + uint32_t len; + uint32_t totalBlocks; + uint32_t totalSubBlocks; + uint32_t offset; + uint64_t size; + uint64_t tombSize; +} SDFInfo; + +typedef struct { + SDFInfo info; + TFILE f; + int fd; + uint8_t state; +} SDFile; + +void tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype); +void tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile); +int tsdbEncodeSDFile(void** buf, SDFile* pDFile); +void* tsdbDecodeSDFile(void* buf, SDFile* pDFile); +int tsdbCreateDFile(SDFile* pDFile, bool updateHeader); +int tsdbUpdateDFileHeader(SDFile* pDFile); +int tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo); +int tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version); + +static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; } + +static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { + ASSERT(!TSDB_FILE_OPENED(pDFile)); + + pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { + if (TSDB_FILE_OPENED(pDFile)) { + close(pDFile->fd); + TSDB_FILE_SET_CLOSED(pDFile); + } +} + +static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t loffset = taosLSeek(TSDB_FILE_FD(pDFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nwrite = taosWrite(pDFile->fd, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) { + pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM)); +} + +static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t toffset; + + if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pDFile->info.size == toffset); + + if (offset) { + *offset = toffset; + } + + if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) { + return -1; + } + + pDFile->info.size += nbyte; + + return (int)nbyte; +} + +static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); } + +static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { + ASSERT(TSDB_FILE_OPENED(pDFile)); + + int64_t nread = taosRead(pDFile->fd, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) { + if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc)); + return 0; +} + +// =============== SDFileSet +typedef struct { + int fid; + int state; + SDFile files[TSDB_FILE_MAX]; +} SDFileSet; + +#define TSDB_FSET_FID(s) ((s)->fid) +#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t)) +#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0)) +#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0)) +#define TSDB_FSET_SET_CLOSED(s) \ + do { \ + for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \ + TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype)); \ + } \ + } while (0); +#define TSDB_FSET_FSYNC(s) \ + do { \ + for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \ + TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype)); \ + } \ + } while (0); + +void tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver); +void tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet); +int tsdbEncodeDFileSet(void** buf, SDFileSet* pSet); +void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet); +int tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet); +void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet); +int tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to); +int tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader); +int tsdbUpdateDFileSetHeader(SDFileSet* pSet); +int tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet* pSet); + +static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) { + tsdbCloseDFileSet(pSet); + return -1; + } + } + return 0; +} + +static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + (void)tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype)); + } +} + +static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) { + tsdbRemoveDFileSet(pDest); + return -1; + } + } + + return 0; +} + +static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) { + *minKey = fid * days * tsMsPerDay[precision]; + *maxKey = *minKey + days * tsMsPerDay[precision] - 1; +} + +static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (TSDB_FILE_IS_BAD(TSDB_DFILE_IN_SET(pSet, ftype))) { + return false; + } + } + + return true; +} + +#endif /* _TS_TSDB_FILE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbLog.h b/src/tsdb/inc/tsdbLog.h new file mode 100644 index 0000000000000000000000000000000000000000..fdd04e968a2ce5345030aa4cef2bb87becd8f3f9 --- /dev/null +++ b/src/tsdb/inc/tsdbLog.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_LOG_H_ +#define _TD_TSDB_LOG_H_ + +extern int32_t tsdbDebugFlag; + +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) + +#endif /* _TD_TSDB_LOG_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h deleted file mode 100644 index 50679749036d7b38e124aad6980cdd44d40998a8..0000000000000000000000000000000000000000 --- a/src/tsdb/inc/tsdbMain.h +++ /dev/null @@ -1,619 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_TSDB_MAIN_H_ -#define _TD_TSDB_MAIN_H_ - -#include "os.h" -#include "hash.h" -#include "tcoding.h" -#include "tglobal.h" -#include "tkvstore.h" -#include "tlist.h" -#include "tlog.h" -#include "tlockfree.h" -#include "tsdb.h" -#include "tskiplist.h" -#include "tutil.h" - -#ifdef __cplusplus -extern "C" { -#endif - -extern int32_t tsdbDebugFlag; - -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) - -#define TSDB_MAX_TABLE_SCHEMAS 16 -#define TSDB_FILE_HEAD_SIZE 512 -#define TSDB_FILE_DELIMITER 0xF00AFA0F -#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF - -#define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax))) - -// NOTE: Any file format change must increase this version number by 1 -// Also, implement the convert function -#define TSDB_FILE_VERSION ((uint32_t)0) - -// Definitions -// ------------------ tsdbMeta.c -typedef struct STable { - STableId tableId; - ETableType type; - tstr* name; // NOTE: there a flexible string here - uint64_t suid; - struct STable* pSuper; // super table pointer - uint8_t numOfSchemas; - STSchema* schema[TSDB_MAX_TABLE_SCHEMAS]; - STSchema* tagSchema; - SKVRow tagVal; - SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index - void* eventHandler; // TODO - void* streamHandler; // TODO - TSKEY lastKey; - SDataRow lastRow; - char* sql; - void* cqhandle; - SRWLatch latch; // TODO: implementa latch functions - T_REF_DECLARE() -} STable; - -typedef struct { - pthread_rwlock_t rwLock; - - int32_t nTables; - int32_t maxTables; - STable** tables; - SList* superList; - SHashObj* uidMap; - SKVStore* pStore; - int maxRowBytes; - int maxCols; -} STsdbMeta; - -// ------------------ tsdbBuffer.c -typedef struct { - int64_t blockId; - int offset; - int remain; - char data[]; -} STsdbBufBlock; - -typedef struct { - pthread_cond_t poolNotEmpty; - int bufBlockSize; - int tBufBlocks; - int nBufBlocks; - int64_t index; - SList* bufBlockList; -} STsdbBufPool; - -// ------------------ tsdbMemTable.c -typedef struct { - STable * pTable; - SSkipListIterator *pIter; -} SCommitIter; - -typedef struct { - uint64_t uid; - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - SSkipList* pData; -} STableData; - -typedef struct { - T_REF_DECLARE() - SRWLatch latch; - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - int32_t maxTables; - STableData** tData; - SList* actList; - SList* extraBuffList; - SList* bufBlockList; -} SMemTable; - -enum { TSDB_UPDATE_META, TSDB_DROP_META }; - -#ifdef WINDOWS -#pragma pack(push ,1) -typedef struct { -#else -typedef struct __attribute__((packed)){ -#endif - char act; - uint64_t uid; -} SActObj; -#ifdef WINDOWS -#pragma pack(pop) -#endif - -typedef struct { - int len; - char cont[]; -} SActCont; - -// ------------------ tsdbFile.c -extern const char* tsdbFileSuffix[]; -typedef enum { - TSDB_FILE_TYPE_HEAD = 0, - TSDB_FILE_TYPE_DATA, - TSDB_FILE_TYPE_LAST, - TSDB_FILE_TYPE_STAT, - TSDB_FILE_TYPE_NHEAD, - TSDB_FILE_TYPE_NDATA, - TSDB_FILE_TYPE_NLAST, - TSDB_FILE_TYPE_NSTAT -} TSDB_FILE_TYPE; - -#ifndef TDINTERNAL -#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1) -#else -#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1) -#endif - -typedef struct { - uint32_t magic; - uint32_t len; - uint32_t totalBlocks; - uint32_t totalSubBlocks; - uint32_t offset; - uint64_t size; // total size of the file - uint64_t tombSize; // unused file size -} STsdbFileInfo; - -typedef struct { - char fname[TSDB_FILENAME_LEN]; - int fd; - - STsdbFileInfo info; -} SFile; - -typedef struct { - int fileId; - int state; // 0 for health, 1 for problem - SFile files[TSDB_FILE_TYPE_MAX]; -} SFileGroup; - -typedef struct { - pthread_rwlock_t fhlock; - - int maxFGroups; - int nFGroups; - SFileGroup* pFGroup; -} STsdbFileH; - -typedef struct { - int direction; - STsdbFileH* pFileH; - int fileId; - int index; -} SFileGroupIter; - -// ------------------ tsdbMain.c -typedef struct { - int32_t totalLen; - int32_t len; - SDataRow row; -} SSubmitBlkIter; - -typedef struct { - int32_t totalLen; - int32_t len; - void * pMsg; -} SSubmitMsgIter; - -typedef struct { - int8_t state; - - char* rootDir; - STsdbCfg config; - STsdbAppH appH; - STsdbStat stat; - STsdbMeta* tsdbMeta; - STsdbBufPool* pPool; - SMemTable* mem; - SMemTable* imem; - STsdbFileH* tsdbFileH; - sem_t readyToCommit; - pthread_mutex_t mutex; - bool repoLocked; - int32_t code; // Commit code -} STsdbRepo; - -// ------------------ tsdbRWHelper.c -typedef struct { - int32_t tid; - uint32_t len; - uint32_t offset; - uint32_t hasLast : 2; - uint32_t numOfBlocks : 30; - uint64_t uid; - TSKEY maxKey; -} SCompIdx; - -typedef struct { - int64_t last : 1; - int64_t offset : 63; - int32_t algorithm : 8; - int32_t numOfRows : 24; - int32_t len; - int32_t keyLen; // key column length, keyOffset = offset+sizeof(SCompData)+sizeof(SCompCol)*numOfCols - int16_t numOfSubBlocks; - int16_t numOfCols; // not including timestamp column - TSKEY keyFirst; - TSKEY keyLast; -} SCompBlock; - -typedef struct { - int32_t delimiter; // For recovery usage - int32_t tid; - uint64_t uid; - SCompBlock blocks[]; -} SCompInfo; - -typedef struct { - int16_t colId; - int32_t len; - int32_t type : 8; - int32_t offset : 24; - int64_t sum; - int64_t max; - int64_t min; - int16_t maxIndex; - int16_t minIndex; - int16_t numOfNull; - char padding[2]; -} SCompCol; - -typedef struct { - int32_t delimiter; // For recovery usage - int32_t numOfCols; // For recovery usage - uint64_t uid; // For recovery usage - SCompCol cols[]; -} SCompData; - -typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; - -typedef struct { - TSKEY minKey; - TSKEY maxKey; - SFileGroup fGroup; - SFile nHeadF; - SFile nLastF; -} SHelperFile; - -typedef struct { - uint64_t uid; - int32_t tid; -} SHelperTable; - -typedef struct { - SCompIdx* pIdxArray; - int numOfIdx; - int curIdx; -} SIdxH; - -typedef struct { - tsdb_rw_helper_t type; - - STsdbRepo* pRepo; - int8_t state; - // For file set usage - SHelperFile files; - SIdxH idxH; - SCompIdx curCompIdx; - void* pWIdx; - // For table set usage - SHelperTable tableInfo; - SCompInfo* pCompInfo; - bool hasOldLastBlock; - // For block set usage - SCompData* pCompData; - SDataCols* pDataCols[2]; - void* pBuffer; // Buffer to hold the whole data block - void* compBuffer; // Buffer for temperary compress/decompress purpose -} SRWHelper; - -typedef struct { - int rowsInserted; - int rowsUpdated; - int rowsDeleteSucceed; - int rowsDeleteFailed; - int nOperations; - TSKEY keyFirst; - TSKEY keyLast; -} SMergeInfo; -// ------------------ tsdbScan.c -typedef struct { - SFileGroup fGroup; - int numOfIdx; - SCompIdx* pCompIdx; - SCompInfo* pCompInfo; - void* pBuf; - FILE* tLogStream; -} STsdbScanHandle; - -// Operations -// ------------------ tsdbMeta.c -#define TSDB_INIT_NTABLES 1024 -#define TABLE_TYPE(t) (t)->type -#define TABLE_NAME(t) (t)->name -#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data -#define TABLE_UID(t) (t)->tableId.uid -#define TABLE_TID(t) (t)->tableId.tid -#define TABLE_SUID(t) (t)->suid -#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) -#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch)) -#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch)) -#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch)) -#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch)) - -STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); -void tsdbFreeMeta(STsdbMeta* pMeta); -int tsdbOpenMeta(STsdbRepo* pRepo); -int tsdbCloseMeta(STsdbRepo* pRepo); -STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); -STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t version); -int tsdbWLockRepoMeta(STsdbRepo* pRepo); -int tsdbRLockRepoMeta(STsdbRepo* pRepo); -int tsdbUnlockRepoMeta(STsdbRepo* pRepo); -void tsdbRefTable(STable* pTable); -void tsdbUnRefTable(STable* pTable); -void tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct); - -static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) { - if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { - return -1; - } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) { - return 1; - } else { - return 0; - } -} - -static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) { - STable* pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; - STSchema* pSchema = NULL; - STSchema* pTSchema = NULL; - - if (lock) TSDB_RLOCK_TABLE(pDTable); - if (version < 0) { // get the latest version of schema - pTSchema = pDTable->schema[pDTable->numOfSchemas - 1]; - } else { // get the schema with version - void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*), - tsdbCompareSchemaVersion, TD_EQ); - if (ptr == NULL) { - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - goto _exit; - } - pTSchema = *(STSchema**)ptr; - } - - ASSERT(pTSchema != NULL); - - if (copy) { - if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - } else { - pSchema = pTSchema; - } - -_exit: - if (lock) TSDB_RUNLOCK_TABLE(pDTable); - return pSchema; -} - -static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) { - return tsdbGetTableSchemaImpl(pTable, false, false, -1); -} - -static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { - if (pTable->type == TSDB_CHILD_TABLE) { // check child table first - STable *pSuper = pTable->pSuper; - if (pSuper == NULL) return NULL; - return pSuper->tagSchema; - } else if (pTable->type == TSDB_SUPER_TABLE) { - return pTable->tagSchema; - } else { - return NULL; - } -} - -static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { - ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow)); - return pTable->lastKey; -} - -// ------------------ tsdbBuffer.c -#define TSDB_BUFFER_RESERVE 1024 // Reseve 1K as commit threshold - -STsdbBufPool* tsdbNewBufPool(); -void tsdbFreeBufPool(STsdbBufPool* pBufPool); -int tsdbOpenBufPool(STsdbRepo* pRepo); -void tsdbCloseBufPool(STsdbRepo* pRepo); -SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); - -// ------------------ tsdbMemTable.c -int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); -void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem); -void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); -int tsdbAsyncCommit(STsdbRepo* pRepo); -int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); -void* tsdbCommitData(STsdbRepo* pRepo); - -static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { - if (pIter == NULL) return NULL; - - SSkipListNode* node = tSkipListIterGet(pIter); - if (node == NULL) return NULL; - - return (SDataRow)SL_GET_NODE_DATA(node); -} - -static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { - SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; - - return dataRowKey(row); -} - -static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { - SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) return TKEY_NULL; - - return dataRowTKey(row); -} - -static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { - ASSERT(pRepo != NULL); - if (pRepo->mem == NULL) return NULL; - - SListNode* pNode = listTail(pRepo->mem->bufBlockList); - if (pNode == NULL) return NULL; - - STsdbBufBlock* pBufBlock = NULL; - tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock)); - - return pBufBlock; -} - -// ------------------ tsdbFile.c -#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) -#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3) -#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId -#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId -#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0) -#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC -#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC - -STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); -void tsdbFreeFileH(STsdbFileH* pFileH); -int tsdbOpenFileH(STsdbRepo* pRepo); -void tsdbCloseFileH(STsdbRepo* pRepo); -SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid); -void tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction); -void tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid); -SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter); -int tsdbOpenFile(SFile* pFile, int oflag); -void tsdbCloseFile(SFile* pFile); -int tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type); -SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags); -void tsdbFitRetention(STsdbRepo* pRepo); -int tsdbUpdateFileHeader(SFile* pFile); -int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); -void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); -void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); -int tsdbLoadFileHeader(SFile* pFile, uint32_t* version); -void tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int64_t* size); -void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); - -// ------------------ tsdbRWHelper.c -#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state -#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set -#define TSDB_HELPER_IDX_LOAD 0x2 // SCompIdx part is loaded -#define TSDB_HELPER_TABLE_SET 0x4 // Table is set -#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded -#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded -#define helperSetState(h, s) (((h)->state) |= (s)) -#define helperClearState(h, s) ((h)->state &= (~(s))) -#define helperHasState(h, s) ((((h)->state) & (s)) == (s)) -#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx) -#define TSDB_MAX_SUBBLOCKS 8 -#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) -#define helperType(h) (h)->type -#define helperRepo(h) (h)->pRepo -#define helperState(h) (h)->state -#define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) -#define helperFileId(h) ((h)->files.fGroup.fileId) -#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) -#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) -#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) -#define helperNewHeadF(h) (&((h)->files.nHeadF)) -#define helperNewLastF(h) (&((h)->files.nLastF)) - -int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo); -int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); -void tsdbDestroyHelper(SRWHelper* pHelper); -void tsdbResetHelper(SRWHelper* pHelper); -int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup); -int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup); -int tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); -int tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey); -int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); -int tsdbWriteCompInfo(SRWHelper* pHelper); -int tsdbWriteCompIdx(SRWHelper* pHelper); -int tsdbLoadCompIdxImpl(SFile* pFile, uint32_t offset, uint32_t len, void* buffer); -int tsdbDecodeSCompIdxImpl(void* buffer, uint32_t len, SCompIdx** ppCompIdx, int* numOfIdx); -int tsdbLoadCompIdx(SRWHelper* pHelper, void* target); -int tsdbLoadCompInfoImpl(SFile* pFile, SCompIdx* pIdx, SCompInfo** ppCompInfo); -int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); -int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target); -void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); -int tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds, - int numOfColIds); -int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo); - -static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) { - if (*(TSKEY*)key1 > *(TSKEY*)key2) { - return 1; - } else if (*(TSKEY*)key1 == *(TSKEY*)key2) { - return 0; - } else { - return -1; - } -} - -// ------------------ tsdbMain.c -#define REPO_ID(r) (r)->config.tsdbId -#define IS_REPO_LOCKED(r) (r)->repoLocked -#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) - -char* tsdbGetMetaFileName(char* rootDir); -void tsdbGetDataFileName(char* rootDir, int vid, int fid, int type, char* fname); -int tsdbLockRepo(STsdbRepo* pRepo); -int tsdbUnlockRepo(STsdbRepo* pRepo); -char* tsdbGetDataDirName(char* rootDir); -int tsdbGetNextMaxTables(int tid); -STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo); -STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo); -int tsdbCheckCommit(STsdbRepo* pRepo); - -// ------------------ tsdbScan.c -int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid); -STsdbScanHandle* tsdbNewScanHandle(); -void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream); -int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid); -int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle); -int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx); -int tsdbCloseScanFile(STsdbScanHandle* pScanHandle); -void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle); - -// ------------------ tsdbCommitQueue.c -int tsdbScheduleCommit(STsdbRepo *pRepo); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h new file mode 100644 index 0000000000000000000000000000000000000000..6046274af40b855ac25dba6e220bf3bcfdc1dcca --- /dev/null +++ b/src/tsdb/inc/tsdbMemTable.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_MEMTABLE_H_ +#define _TD_TSDB_MEMTABLE_H_ + +typedef struct { + int rowsInserted; + int rowsUpdated; + int rowsDeleteSucceed; + int rowsDeleteFailed; + int nOperations; + TSKEY keyFirst; + TSKEY keyLast; +} SMergeInfo; + +typedef struct { + STable * pTable; + SSkipListIterator *pIter; +} SCommitIter; + +struct STableData { + uint64_t uid; + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfRows; + SSkipList* pData; + T_REF_DECLARE() +}; + +enum { TSDB_UPDATE_META, TSDB_DROP_META }; + +#ifdef WINDOWS +#pragma pack(push ,1) +typedef struct { +#else +typedef struct __attribute__((packed)){ +#endif + char act; + uint64_t uid; +} SActObj; +#ifdef WINDOWS +#pragma pack(pop) +#endif + +typedef struct { + int len; + char cont[]; +} SActCont; + +int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); +int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); +int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot, SArray* pATable); +void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot); +void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); +int tsdbAsyncCommit(STsdbRepo* pRepo); +int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, + TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); +void* tsdbCommitData(STsdbRepo* pRepo); + +static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { + if (pIter == NULL) return NULL; + + SSkipListNode* node = tSkipListIterGet(pIter); + if (node == NULL) return NULL; + + return (SDataRow)SL_GET_NODE_DATA(node); +} + +static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { + SDataRow row = tsdbNextIterRow(pIter); + if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; + + return dataRowKey(row); +} + +static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { + SDataRow row = tsdbNextIterRow(pIter); + if (row == NULL) return TKEY_NULL; + + return dataRowTKey(row); +} + +#endif /* _TD_TSDB_MEMTABLE_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h new file mode 100644 index 0000000000000000000000000000000000000000..cc916fa689a89f96e36c4419f1b2d413516ed5fc --- /dev/null +++ b/src/tsdb/inc/tsdbMeta.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_META_H_ +#define _TD_TSDB_META_H_ + +#define TSDB_MAX_TABLE_SCHEMAS 16 + +typedef struct STable { + STableId tableId; + ETableType type; + tstr* name; // NOTE: there a flexible string here + uint64_t suid; + struct STable* pSuper; // super table pointer + uint8_t numOfSchemas; + STSchema* schema[TSDB_MAX_TABLE_SCHEMAS]; + STSchema* tagSchema; + SKVRow tagVal; + SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index + void* eventHandler; // TODO + void* streamHandler; // TODO + TSKEY lastKey; + SDataRow lastRow; + char* sql; + void* cqhandle; + SRWLatch latch; // TODO: implementa latch functions + T_REF_DECLARE() +} STable; + +typedef struct { + pthread_rwlock_t rwLock; + + int32_t nTables; + int32_t maxTables; + STable** tables; + SList* superList; + SHashObj* uidMap; + int maxRowBytes; + int maxCols; +} STsdbMeta; + +#define TSDB_INIT_NTABLES 1024 +#define TABLE_TYPE(t) (t)->type +#define TABLE_NAME(t) (t)->name +#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data +#define TABLE_UID(t) (t)->tableId.uid +#define TABLE_TID(t) (t)->tableId.tid +#define TABLE_SUID(t) (t)->suid +// #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) +#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch)) +#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch)) +#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch)) +#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch)) + +STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); +void tsdbFreeMeta(STsdbMeta* pMeta); +int tsdbOpenMeta(STsdbRepo* pRepo); +int tsdbCloseMeta(STsdbRepo* pRepo); +STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); +STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t version); +int tsdbWLockRepoMeta(STsdbRepo* pRepo); +int tsdbRLockRepoMeta(STsdbRepo* pRepo); +int tsdbUnlockRepoMeta(STsdbRepo* pRepo); +void tsdbRefTable(STable* pTable); +void tsdbUnRefTable(STable* pTable); +void tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct); +int tsdbRestoreTable(STsdbRepo* pRepo, void* cont, int contLen); +void tsdbOrgMeta(STsdbRepo* pRepo); + +static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) { + if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { + return -1; + } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) { + return 1; + } else { + return 0; + } +} + +static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) { + STable* pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; + STSchema* pSchema = NULL; + STSchema* pTSchema = NULL; + + if (lock) TSDB_RLOCK_TABLE(pDTable); + if (version < 0) { // get the latest version of schema + pTSchema = pDTable->schema[pDTable->numOfSchemas - 1]; + } else { // get the schema with version + void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*), + tsdbCompareSchemaVersion, TD_EQ); + if (ptr == NULL) { + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + goto _exit; + } + pTSchema = *(STSchema**)ptr; + } + + ASSERT(pTSchema != NULL); + + if (copy) { + if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + } else { + pSchema = pTSchema; + } + +_exit: + if (lock) TSDB_RUNLOCK_TABLE(pDTable); + return pSchema; +} + +static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) { + return tsdbGetTableSchemaImpl(pTable, false, false, -1); +} + +static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) { + if (pTable->type == TSDB_CHILD_TABLE) { // check child table first + STable *pSuper = pTable->pSuper; + if (pSuper == NULL) return NULL; + return pSuper->tagSchema; + } else if (pTable->type == TSDB_SUPER_TABLE) { + return pTable->tagSchema; + } else { + return NULL; + } +} + +static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) { + ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow)); + return pTable->lastKey; +} + +#endif /* _TD_TSDB_META_H_ */ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbReadImpl.h b/src/tsdb/inc/tsdbReadImpl.h new file mode 100644 index 0000000000000000000000000000000000000000..e7840d9723f7935c69f70f1c236a5ed49c82c146 --- /dev/null +++ b/src/tsdb/inc/tsdbReadImpl.h @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_READ_IMPL_H_ +#define _TD_TSDB_READ_IMPL_H_ + +typedef struct SReadH SReadH; + +typedef struct { + int32_t tid; + uint32_t len; + uint32_t offset; + uint32_t hasLast : 2; + uint32_t numOfBlocks : 30; + uint64_t uid; + TSKEY maxKey; +} SBlockIdx; + +typedef struct { + int64_t last : 1; + int64_t offset : 63; + int32_t algorithm : 8; + int32_t numOfRows : 24; + int32_t len; + int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols + int16_t numOfSubBlocks; + int16_t numOfCols; // not including timestamp column + TSKEY keyFirst; + TSKEY keyLast; +} SBlock; + +typedef struct { + int32_t delimiter; // For recovery usage + int32_t tid; + uint64_t uid; + SBlock blocks[]; +} SBlockInfo; + +typedef struct { + int16_t colId; + int32_t len; + uint32_t type : 8; + uint32_t offset : 24; + int64_t sum; + int64_t max; + int64_t min; + int16_t maxIndex; + int16_t minIndex; + int16_t numOfNull; + uint8_t offsetH; + char padding[1]; +} SBlockCol; + +// Code here just for back-ward compatibility +static FORCE_INLINE void tsdbSetBlockColOffset(SBlockCol *pBlockCol, uint32_t offset) { + pBlockCol->offset = offset & ((((uint32_t)1) << 24) - 1); + pBlockCol->offsetH = (uint8_t)(offset >> 24); +} + +static FORCE_INLINE uint32_t tsdbGetBlockColOffset(SBlockCol *pBlockCol) { + uint32_t offset1 = pBlockCol->offset; + uint32_t offset2 = pBlockCol->offsetH; + return (offset1 | (offset2 << 24)); +} + +typedef struct { + int32_t delimiter; // For recovery usage + int32_t numOfCols; // For recovery usage + uint64_t uid; // For recovery usage + SBlockCol cols[]; +} SBlockData; + +struct SReadH { + STsdbRepo * pRepo; + SDFileSet rSet; // FSET to read + SArray * aBlkIdx; // SBlockIdx array + STable * pTable; // table to read + SBlockIdx * pBlkIdx; // current reading table SBlockIdx + int cidx; + SBlockInfo *pBlkInfo; + SBlockData *pBlkData; // Block info + SDataCols * pDCols[2]; + void * pBuf; // buffer + void * pCBuf; // compression buffer +}; + +#define TSDB_READ_REPO(rh) ((rh)->pRepo) +#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) +#define TSDB_READ_FSET(rh) (&((rh)->rSet)) +#define TSDB_READ_TABLE(rh) ((rh)->pTable) +#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD) +#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA) +#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST) +#define TSDB_READ_BUF(rh) ((rh)->pBuf) +#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf) + +#define TSDB_BLOCK_STATIS_SIZE(ncols) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM)) + +int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo); +void tsdbDestroyReadH(SReadH *pReadh); +int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet); +void tsdbCloseAndUnsetFSet(SReadH *pReadh); +int tsdbLoadBlockIdx(SReadH *pReadh); +int tsdbSetReadTable(SReadH *pReadh, STable *pTable); +int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget); +int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); +int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds); +int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); +int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); +void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols); + +static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) { + void * pBuf = *ppBuf; + size_t tsize = taosTSizeof(pBuf); + + if (tsize < size) { + if (tsize == 0) tsize = 1024; + + while (tsize < size) { + tsize *= 2; + } + + *ppBuf = taosTRealloc(pBuf, tsize); + if (*ppBuf == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + + return 0; +} + +#endif /*_TD_TSDB_READ_IMPL_H_*/ \ No newline at end of file diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h new file mode 100644 index 0000000000000000000000000000000000000000..074ff20f2298918f1fa0698be0a291081ead8f05 --- /dev/null +++ b/src/tsdb/inc/tsdbint.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TSDB_INT_H_ +#define _TD_TSDB_INT_H_ + +// // TODO: remove the include +// #include +// #include +// #include +// #include +// #include +// #include +// #include +// #include + +#include "os.h" +#include "tlog.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tchecksum.h" +#include "tskiplist.h" +#include "tdataformat.h" +#include "tcoding.h" +#include "tscompression.h" +#include "tlockfree.h" +#include "tlist.h" +#include "hash.h" +#include "tarray.h" +#include "tfs.h" +#include "tsocket.h" + +#include "tsdb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Log +#include "tsdbLog.h" +// Meta +#include "tsdbMeta.h" +// Buffer +#include "tsdbBuffer.h" +// MemTable +#include "tsdbMemTable.h" +// File +#include "tsdbFile.h" +// FS +#include "tsdbFS.h" +// ReadImpl +#include "tsdbReadImpl.h" +// Commit +#include "tsdbCommit.h" +// Commit Queue +#include "tsdbCommitQueue.h" +// Main definitions +struct STsdbRepo { + uint8_t state; + + STsdbCfg config; + STsdbAppH appH; + STsdbStat stat; + STsdbMeta* tsdbMeta; + STsdbBufPool* pPool; + SMemTable* mem; + SMemTable* imem; + STsdbFS* fs; + tsem_t readyToCommit; + pthread_mutex_t mutex; + bool repoLocked; + int32_t code; // Commit code +}; + +#define REPO_ID(r) (r)->config.tsdbId +#define REPO_CFG(r) (&((r)->config)) +#define REPO_FS(r) ((r)->fs) +#define IS_REPO_LOCKED(r) (r)->repoLocked +#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) + +int tsdbLockRepo(STsdbRepo* pRepo); +int tsdbUnlockRepo(STsdbRepo* pRepo); +STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo); +int tsdbCheckCommit(STsdbRepo* pRepo); +int tsdbRestoreInfo(STsdbRepo* pRepo); +void tsdbGetRootDir(int repoid, char dirName[]); +void tsdbGetDataDir(int repoid, char dirName[]); + +static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { + ASSERT(pRepo != NULL); + if (pRepo->mem == NULL) return NULL; + + SListNode* pNode = listTail(pRepo->mem->bufBlockList); + if (pNode == NULL) return NULL; + + STsdbBufBlock* pBufBlock = NULL; + tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock)); + + return pBufBlock; +} + +static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { + ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); + int maxTables = TSDB_INIT_NTABLES; + while (true) { + maxTables = MIN(maxTables, TSDB_MAX_TABLES); + if (tid <= maxTables) break; + maxTables *= 2; + } + + return maxTables + 1; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _TD_TSDB_INT_H_ */ \ No newline at end of file diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 7cea27658c80d689972e3cb0f5dda3269a34b720..1798a21b9963c7641dd99dc7fa11a5dd977e0e3c 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "tsdb.h" -#include "tsdbMain.h" +#include "tsdbint.h" #define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 696270d67086286dd3e9d86a69a5cc9cdc4b53eb..4351cce51c09f86fc06cee59c94b800c44e60b1c 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -12,23 +12,85 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "tsdbMain.h" +#include "tsdbint.h" + +#define TSDB_MAX_SUBBLOCKS 8 +static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { + if (key < 0) { + return (int)((key + 1) / tsMsPerDay[precision] / days - 1); + } else { + return (int)((key / tsMsPerDay[precision] / days)); + } +} + +typedef struct { + SRtn rtn; // retention snapshot + SFSIter fsIter; // tsdb file iterator + int niters; // memory iterators + SCommitIter *iters; + bool isRFileSet; // read and commit FSET + SReadH readh; + SDFileSet wSet; + bool isDFileSame; + bool isLFileSame; + TSKEY minKey; + TSKEY maxKey; + SArray * aBlkIdx; // SBlockIdx array + STable * pTable; + SArray * aSupBlk; // Table super-block array + SArray * aSubBlk; // table sub-block array + SDataCols * pDataCols; +} SCommitH; + +#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh)) +#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh))) +#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet)) +#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable) +#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD) +#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA) +#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST) +#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh)) +#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh)) +#define TSDB_COMMIT_DEFAULT_ROWS(ch) (TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock * 4 / 5) +#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) -static int tsdbCommitTSData(STsdbRepo *pRepo); static int tsdbCommitMeta(STsdbRepo *pRepo); +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen); +static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); +static int tsdbCommitTSData(STsdbRepo *pRepo); +static void tsdbStartCommit(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo, int eno); -static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); -static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); -static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static int tsdbCreateCommitIters(SCommitH *pCommith); +static void tsdbDestroyCommitIters(SCommitH *pCommith); +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key); +static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo); +static void tsdbDestroyCommitH(SCommitH *pCommith); +static int tsdbGetFidLevel(int fid, SRtn *pRtn); +static int tsdbNextCommitFid(SCommitH *pCommith); +static int tsdbCommitToTable(SCommitH *pCommith, int tid); +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); +static int tsdbComparKeyBlock(const void *arg1, const void *arg2); +static int tsdbWriteBlockInfo(SCommitH *pCommih); +static int tsdbWriteBlockIdx(SCommitH *pCommih); +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData); +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx); +static int tsdbMoveBlock(SCommitH *pCommith, int bidx); +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks); +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, + bool isLastOneBlock); +static void tsdbResetCommitFile(SCommitH *pCommith); +static void tsdbResetCommitTable(SCommitH *pCommith); +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError); +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo); +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update); +static int tsdbApplyRtn(STsdbRepo *pRepo); +static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn); void *tsdbCommitData(STsdbRepo *pRepo) { - SMemTable * pMem = pRepo->imem; - - tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d", - REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList)); - - pRepo->code = TSDB_CODE_SUCCESS; + tsdbStartCommit(pRepo); // Commit to update meta file if (tsdbCommitMeta(pRepo) < 0) { @@ -42,304 +104,1393 @@ void *tsdbCommitData(STsdbRepo *pRepo) { goto _err; } - tsdbFitRetention(pRepo); - - tsdbInfo("vgId:%d commit over, succeed", REPO_ID(pRepo)); tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS); - return NULL; _err: ASSERT(terrno != TSDB_CODE_SUCCESS); pRepo->code = terrno; - tsdbInfo("vgId:%d commit over, failed", REPO_ID(pRepo)); - tsdbEndCommit(pRepo, terrno); + tsdbEndCommit(pRepo, terrno); return NULL; } -static int tsdbCommitTSData(STsdbRepo *pRepo) { - SMemTable * pMem = pRepo->imem; - SDataCols * pDataCols = NULL; - STsdbMeta * pMeta = pRepo->tsdbMeta; - SCommitIter *iters = NULL; - SRWHelper whelper = {0}; - STsdbCfg * pCfg = &(pRepo->config); - - if (pMem->numOfRows <= 0) return 0; - - iters = tsdbCreateCommitIters(pRepo); - if (iters == NULL) { - tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; +// =================== Commit Meta Data +static int tsdbCommitMeta(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SMemTable *pMem = pRepo->imem; + SMFile * pOMFile = pfs->cstatus->pmf; + SMFile mf; + SActObj * pAct = NULL; + SActCont * pCont = NULL; + SListNode *pNode = NULL; + SDiskID did; + + ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); + + if (listNEles(pMem->actList) <= 0) { + // no meta data to commit, just keep the old meta file + tsdbUpdateMFile(pfs, pOMFile); + return 0; + } else { + // Create/Open a meta file or open the existing file + if (pOMFile == NULL) { + // Create a new meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + + if (tsdbCreateMFile(&mf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); + } else { + tsdbInitMFileEx(&mf, pOMFile); + if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } } - if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { - tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + // Loop to write + while ((pNode = tdListPopHead(pMem->actList)) != NULL) { + pAct = (SActObj *)pNode->data; + if (pAct->act == TSDB_UPDATE_META) { + pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); + if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + tsdbCloseMFile(&mf); + (void)tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + } else if (pAct->act == TSDB_DROP_META) { + if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) { + tsdbError("vgId:%d failed to drop META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + tsdbCloseMFile(&mf); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; + } + } else { + ASSERT(false); + } } - if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", - REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); - goto _err; + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbError("vgId:%d failed to update META file header since %s, revert it", REPO_ID(pRepo), tstrerror(terrno)); + tsdbApplyMFileChange(&mf, pOMFile); + // TODO: need to reload metaCache + return -1; } - int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision)); - int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision)); + TSDB_FILE_FSYNC(&mf); + tsdbCloseMFile(&mf); + tsdbUpdateMFile(pfs, &mf); - // Loop to commit to each file - for (int fid = sfid; fid <= efid; fid++) { - if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { - tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - goto _err; - } + return 0; +} + +int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pRecord->uid); + tlen += taosEncodeFixedI64(buf, pRecord->offset); + tlen += taosEncodeFixedI64(buf, pRecord->size); + + return tlen; +} + +void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) { + buf = taosDecodeFixedU64(buf, &(pRecord->uid)); + buf = taosDecodeFixedI64(buf, &(pRecord->offset)); + buf = taosDecodeFixedI64(buf, &(pRecord->size)); + + return buf; +} + +void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + TSKEY minKey, midKey, maxKey, now; + + now = taosGetTimestamp(pCfg->precision); + minKey = now - pCfg->keep * tsMsPerDay[pCfg->precision]; + midKey = now - pCfg->keep2 * tsMsPerDay[pCfg->precision]; + maxKey = now - pCfg->keep1 * tsMsPerDay[pCfg->precision]; + + pRtn->minKey = minKey; + pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision)); + pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision)); + tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey, + pRtn->minFid, pRtn->midFid, pRtn->maxFid); +} + +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) { + char buf[64] = "\0"; + void * pBuf = buf; + SKVRecord rInfo; + int64_t offset; + + // Seek to end of meta file + offset = tsdbSeekMFile(pMFile, 0, SEEK_END); + if (offset < 0) { + return -1; + } + + rInfo.offset = offset; + rInfo.uid = uid; + rInfo.size = contLen; + + int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo); + if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) { + return -1; + } + + if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) { + return -1; } - tdFreeDataCols(pDataCols); - tsdbDestroyCommitIters(iters, pMem->maxTables); - tsdbDestroyHelper(&whelper); + tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); + SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid)); + if (pRecord != NULL) { + pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); + } else { + pMFile->info.nRecords++; + } + taosHashPut(pfs->metaCache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); return 0; +} -_err: - tdFreeDataCols(pDataCols); - tsdbDestroyCommitIters(iters, pMem->maxTables); - tsdbDestroyHelper(&whelper); +static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { + SKVRecord rInfo = {0}; + char buf[128] = "\0"; + + SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid)); + if (pRecord == NULL) { + tsdbError("failed to drop META record with key %" PRIu64 " since not find", uid); + return -1; + } + + rInfo.offset = -pRecord->offset; + rInfo.uid = pRecord->uid; + rInfo.size = pRecord->size; - return -1; + void *pBuf = buf; + tsdbEncodeKVRecord(&pBuf, &rInfo); + + if (tsdbAppendMFile(pMFile, buf, sizeof(SKVRecord), NULL) < 0) { + return -1; + } + + pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, sizeof(SKVRecord)); + pMFile->info.nDels++; + pMFile->info.nRecords--; + pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); + + taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid)); + return 0; } -static int tsdbCommitMeta(STsdbRepo *pRepo) { +// =================== Commit Time-Series Data +static int tsdbCommitTSData(STsdbRepo *pRepo) { SMemTable *pMem = pRepo->imem; - STsdbMeta *pMeta = pRepo->tsdbMeta; - SActObj * pAct = NULL; - SActCont * pCont = NULL; + SCommitH commith; + SDFileSet *pSet = NULL; + int fid; - if (listNEles(pMem->actList) <= 0) return 0; + memset(&commith, 0, sizeof(commith)); - if (tdKVStoreStartCommit(pMeta->pStore) < 0) { - tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + if (pMem->numOfRows <= 0) { + // No memory data, just apply retention on each file on disk + if (tsdbApplyRtn(pRepo) < 0) { + return -1; + } + return 0; } - SListNode *pNode = NULL; + // Resource initialization + if (tsdbInitCommitH(&commith, pRepo) < 0) { + return -1; + } - while ((pNode = tdListPopHead(pMem->actList)) != NULL) { - pAct = (SActObj *)pNode->data; - if (pAct->act == TSDB_UPDATE_META) { - pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); - if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { - tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, - tstrerror(terrno)); - tdKVStoreEndCommit(pMeta->pStore); - goto _err; - } - } else if (pAct->act == TSDB_DROP_META) { - if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) { - tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, - tstrerror(terrno)); - tdKVStoreEndCommit(pMeta->pStore); - goto _err; - } + // Skip expired memory data and expired FSET + tsdbSeekCommitIter(&commith, commith.rtn.minKey); + while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) { + if (pSet->fid < commith.rtn.minFid) { + tsdbInfo("vgId:%d FSET %d on level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); } else { - ASSERT(false); + break; } } - if (tdKVStoreEndCommit(pMeta->pStore) < 0) { - tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + // Loop to commit to each file + fid = tsdbNextCommitFid(&(commith)); + while (true) { + // Loop over both on disk and memory + if (pSet == NULL && fid == TSDB_IVLD_FID) break; + + if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) { + // Only has existing FSET but no memory data to commit in this + // existing FSET, only check if file in correct retention + if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + pSet = tsdbFSIterNext(&(commith.fsIter)); + } else { + // Has memory data to commit + SDFileSet *pCSet; + int cfid; + + if (pSet == NULL || pSet->fid > fid) { + // Commit to a new FSET with fid: fid + pCSet = NULL; + cfid = fid; + } else { + // Commit to an existing FSET + pCSet = pSet; + cfid = pSet->fid; + pSet = tsdbFSIterNext(&(commith.fsIter)); + } + + if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) { + tsdbDestroyCommitH(&commith); + return -1; + } + + fid = tsdbNextCommitFid(&commith); + } } + tsdbDestroyCommitH(&commith); return 0; +} -_err: - return -1; +static void tsdbStartCommit(STsdbRepo *pRepo) { + SMemTable *pMem = pRepo->imem; + + ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0); + + tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d", + REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList)); + + tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd); + + pRepo->code = TSDB_CODE_SUCCESS; } static void tsdbEndCommit(STsdbRepo *pRepo, int eno) { + if (eno != TSDB_CODE_SUCCESS) { + tsdbEndFSTxnWithError(REPO_FS(pRepo)); + } else { + tsdbEndFSTxn(pRepo); + } + + tsdbInfo("vgId:%d commit over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno); + SMemTable *pIMem = pRepo->imem; - tsdbLockRepo(pRepo); + (void)tsdbLockRepo(pRepo); pRepo->imem = NULL; - tsdbUnlockRepo(pRepo); + (void)tsdbUnlockRepo(pRepo); tsdbUnRefMemTable(pRepo, pIMem); - sem_post(&(pRepo->readyToCommit)); + tsem_post(&(pRepo->readyToCommit)); } -static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { +#if 0 +static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { for (int i = 0; i < nIters; i++) { TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); - if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1; + if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true; + } + return false; +} +#endif + +static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + + ASSERT(pSet == NULL || pSet->fid == fid); + + tsdbResetCommitFile(pCommith); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey)); + + // Set and open files + if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) { + return -1; } + + // Loop to commit each table data + for (int tid = 1; tid < pCommith->niters; tid++) { + SCommitIter *pIter = pCommith->iters + tid; + + if (pIter->pTable == NULL) continue; + + if (tsdbCommitToTable(pCommith, tid) < 0) { + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + } + + if (tsdbWriteBlockIdx(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockIdx part to FSET %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + if (tsdbUpdateDFileSetHeader(&(pCommith->wSet)) < 0) { + tsdbError("vgId:%d failed to update FSET %d header since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + tsdbCloseCommitFile(pCommith, true); + // revert the file change + tsdbApplyDFileSetChange(TSDB_COMMIT_WRITE_FSET(pCommith), pSet); + return -1; + } + + // Close commit file + tsdbCloseCommitFile(pCommith, false); + + if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) { + return -1; + } + return 0; } -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) { - char * dataDir = NULL; - STsdbCfg * pCfg = &pRepo->config; - STsdbFileH *pFileH = pRepo->tsdbFileH; - SFileGroup *pGroup = NULL; - SMemTable * pMem = pRepo->imem; - bool newLast = false; +static int tsdbCreateCommitIters(SCommitH *pCommith) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + SMemTable *pMem = pRepo->imem; + STsdbMeta *pMeta = pRepo->tsdbMeta; + + pCommith->niters = pMem->maxTables; + pCommith->iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); + if (pCommith->iters == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } - TSKEY minKey = 0, maxKey = 0; - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + if (tsdbRLockRepoMeta(pRepo) < 0) return -1; - // Check if there are data to commit to this file - int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey); - if (!hasDataToCommit) { - tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid); - return 0; + // reference all tables + for (int i = 0; i < pMem->maxTables; i++) { + if (pMeta->tables[i] != NULL) { + tsdbRefTable(pMeta->tables[i]); + pCommith->iters[i].pTable = pMeta->tables[i]; + } + } + + if (tsdbUnlockRepoMeta(pRepo) < 0) return -1; + + for (int i = 0; i < pMem->maxTables; i++) { + if ((pCommith->iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && + (TABLE_UID(pCommith->iters[i].pTable) == pMem->tData[i]->uid)) { + if ((pCommith->iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + tSkipListIterNext(pCommith->iters[i].pIter); + } + } + + return 0; +} + +static void tsdbDestroyCommitIters(SCommitH *pCommith) { + if (pCommith->iters == NULL) return; + + for (int i = 1; i < pCommith->niters; i++) { + if (pCommith->iters[i].pTable != NULL) { + tsdbUnRefTable(pCommith->iters[i].pTable); + tSkipListDestroyIter(pCommith->iters[i].pIter); + } + } + + free(pCommith->iters); + pCommith->iters = NULL; + pCommith->niters = 0; +} + +// Skip all keys until key (not included) +static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL); + } +} + +static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) { + STsdbCfg *pCfg = REPO_CFG(pRepo); + + memset(pCommith, 0, sizeof(*pCommith)); + tsdbGetRtnSnap(pRepo, &(pCommith->rtn)); + + TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith)); + + // Init read handle + if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) { + return -1; + } + + // Init file iterator + tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD); + + if (tsdbCreateCommitIters(pCommith) < 0) { + tsdbDestroyCommitH(pCommith); + return -1; } - // Create and open files for commit - dataDir = tsdbGetDataDirName(pRepo->rootDir); - if (dataDir == NULL) { + pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + if (pCommith->aBlkIdx == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); return -1; } - if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) { - tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - goto _err; + pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSupBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; } - // Open files for write/read - if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { - tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock)); + if (pCommith->aSubBlk == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; } - newLast = TSDB_NLAST_FILE_OPENED(pHelper); + pCommith->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + if (pCommith->pDataCols == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyCommitH(pCommith); + return -1; + } - if (tsdbLoadCompIdx(pHelper, NULL) < 0) { - tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + return 0; +} + +static void tsdbDestroyCommitH(SCommitH *pCommith) { + pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols); + pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk); + pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk); + pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx); + tsdbDestroyCommitIters(pCommith); + tsdbDestroyReadH(&(pCommith->readh)); + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static int tsdbNextCommitFid(SCommitH *pCommith) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int fid = TSDB_IVLD_FID; + + for (int i = 0; i < pCommith->niters; i++) { + SCommitIter *pIter = pCommith->iters + i; + if (pIter->pTable == NULL || pIter->pIter == NULL) continue; + + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); + if (nextKey == TSDB_DATA_TIMESTAMP_NULL) { + continue; + } else { + int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision)); + if (fid == TSDB_IVLD_FID || fid > tfid) { + fid = tfid; + } + } } - // Loop to commit data in each table - for (int tid = 1; tid < pMem->maxTables; tid++) { - SCommitIter *pIter = iters + tid; - if (pIter->pTable == NULL) continue; + return fid; +} - TSDB_RLOCK_TABLE(pIter->pTable); +static int tsdbCommitToTable(SCommitH *pCommith, int tid) { + SCommitIter *pIter = pCommith->iters + tid; + TSKEY nextKey = tsdbNextIterKey(pIter->pIter); - if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err; + tsdbResetCommitTable(pCommith); - if (pIter->pIter != NULL) { - if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + TSDB_RLOCK_TABLE(pIter->pTable); + + // Set commit table + if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + // No disk data and no memory data, just return + if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return 0; + } + + // Must has disk data or has memory data + int nBlocks; + int bidx = 0; + SBlock *pBlock; + + if (pCommith->readh.pBlkIdx) { + if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + + nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + } else { + nBlocks = 0; + } + + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + + while (true) { + if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break; + + if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) || + (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) { + if (tsdbMoveBlock(pCommith, bidx) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; } - if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) { + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) { + // merge pBlock data and memory data + if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) { TSDB_RUNLOCK_TABLE(pIter->pTable); - tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo), - TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable), - tstrerror(terrno)); - goto _err; + return -1; } + + bidx++; + if (bidx < nBlocks) { + pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + } else { + pBlock = NULL; + } + nextKey = tsdbNextIterKey(pIter->pIter); + } else { + // Only commit memory data + if (pBlock == NULL) { + if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + } else { + if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) { + TSDB_RUNLOCK_TABLE(pIter->pTable); + return -1; + } + } + nextKey = tsdbNextIterKey(pIter->pIter); } + } - TSDB_RUNLOCK_TABLE(pIter->pTable); + TSDB_RUNLOCK_TABLE(pIter->pTable); + + if (tsdbWriteBlockInfo(pCommith) < 0) { + tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith), + TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno)); + return -1; + } + + return 0; +} + +static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) { + STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + + pCommith->pTable = pTable; + + if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } - // Move the last block to the new .l file if neccessary - if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { - tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + if (pCommith->isRFileSet) { + if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) { + return -1; } + } else { + pCommith->readh.pBlkIdx = NULL; + } + return 0; +} + +static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { + TSKEY key = *(TSKEY *)arg1; + SBlock *pBlock = (SBlock *)arg2; + + if (key < pBlock->keyFirst) { + return -1; + } else if (key > pBlock->keyLast) { + return 1; + } else { + return 0; + } +} + +static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, + bool isSuper) { + STsdbRepo * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlockData *pBlockData; + int64_t offset = 0; + STable * pTable = TSDB_COMMIT_TABLE(pCommith); + int rowsToWrite = pDataCols->numOfRows; + + ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); + ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock); + + // Make buffer space + if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommith)), TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) { + return -1; + } + pBlockData = (SBlockData *)TSDB_COMMIT_BUF(pCommith); - // Write the SCompBlock part - if (tsdbWriteCompInfo(pHelper) < 0) { - tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + // Get # of cols not all NULL(not including key column) + int nColsNotAllNull = 0; + for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; + + if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it + continue; + } + + memset(pBlockCol, 0, sizeof(*pBlockCol)); + + pBlockCol->colId = pDataCol->colId; + pBlockCol->type = pDataCol->type; + if (tDataTypes[pDataCol->type].statisFunc) { + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), + &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), + &(pBlockCol->numOfNull)); } + nColsNotAllNull++; } - if (tsdbWriteCompIdx(pHelper) < 0) { - tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); - goto _err; + ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); + + // Compress the data if neccessary + int tcol = 0; // counter of not all NULL and written columns + uint32_t toffset = 0; + int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull); + int32_t lsize = tsize; + int32_t keyLen = 0; + for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + // All not NULL columns finish + if (ncol != 0 && tcol >= nColsNotAllNull) break; + + SDataCol * pDataCol = pDataCols->cols + ncol; + SBlockCol *pBlockCol = pBlockData->cols + tcol; + + if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; + + int32_t flen; // final length + int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); + void * tptr; + + // Make room + if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommith)), lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + return -1; + } + pBlockData = (SBlockData *)TSDB_COMMIT_BUF(pCommith); + pBlockCol = pBlockData->cols + tcol; + tptr = POINTER_SHIFT(pBlockData, lsize); + + if (pCfg->compression == TWO_STAGE_COMP && + tsdbMakeRoom((void **)(&TSDB_COMMIT_COMP_BUF(pCommith)), tlen + COMP_OVERFLOW_BYTES) < 0) { + return -1; + } + + // Compress or just copy + if (pCfg->compression) { + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, + TSDB_COMMIT_COMP_BUF(pCommith), tlen + COMP_OVERFLOW_BYTES); + } else { + flen = tlen; + memcpy(tptr, pDataCol->pData, flen); + } + + // Add checksum + ASSERT(flen > 0); + flen += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); + + if (ncol != 0) { + tsdbSetBlockColOffset(pBlockCol, toffset); + pBlockCol->len = flen; + tcol++; + } else { + keyLen = flen; + } + + toffset += flen; + lsize += flen; + } + + pBlockData->delimiter = TSDB_FILE_DELIMITER; + pBlockData->uid = TABLE_UID(pTable); + pBlockData->numOfCols = nColsNotAllNull; + + taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize); + tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM))); + + // Write the whole block to file + if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) { + return -1; + } + + // Update pBlock membership vairables + pBlock->last = isLast; + pBlock->offset = offset; + pBlock->algorithm = pCfg->compression; + pBlock->numOfRows = rowsToWrite; + pBlock->len = lsize; + pBlock->keyLen = keyLen; + pBlock->numOfSubBlocks = isSuper ? 1 : 0; + pBlock->numOfCols = nColsNotAllNull; + pBlock->keyFirst = dataColsKeyFirst(pDataCols); + pBlock->keyLast = dataColsKeyLast(pDataCols); + + tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, + REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, + pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); + + return 0; +} + +static int tsdbWriteBlockInfo(SCommitH *pCommih) { + SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + SBlockIdx blkIdx; + STable * pTable = TSDB_COMMIT_TABLE(pCommih); + SBlock * pBlock; + size_t nSupBlocks; + size_t nSubBlocks; + uint32_t tlen; + SBlockInfo *pBlkInfo; + int64_t offset; + + nSupBlocks = taosArrayGetSize(pCommih->aSupBlk); + nSubBlocks = taosArrayGetSize(pCommih->aSubBlk); + + if (nSupBlocks <= 0) { + // No data (data all deleted) + return 0; + } + + tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM)); + + // Write SBlockInfo part + if (tsdbMakeRoom((void **)(&(TSDB_COMMIT_BUF(pCommih))), tlen) < 0) return -1; + pBlkInfo = TSDB_COMMIT_BUF(pCommih); + + pBlkInfo->delimiter = TSDB_FILE_DELIMITER; + pBlkInfo->tid = TABLE_TID(pTable); + pBlkInfo->uid = TABLE_UID(pTable); + + memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pCommih->aSupBlk, 0), nSupBlocks * sizeof(SBlock)); + if (nSubBlocks > 0) { + memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pCommih->aSubBlk, 0), nSubBlocks * sizeof(SBlock)); + + for (int i = 0; i < nSupBlocks; i++) { + pBlock = pBlkInfo->blocks + i; + + if (pBlock->numOfSubBlocks > 1) { + pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks); + } + } + } + + taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen); + + if (tsdbAppendDFile(pHeadf, TSDB_COMMIT_BUF(pCommih), tlen, &offset) < 0) { + return -1; } - tfree(dataDir); - tsdbCloseHelperFile(pHelper, 0, pGroup); + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM))); - pthread_rwlock_wrlock(&(pFileH->fhlock)); + // Set blkIdx + pBlock = taosArrayGet(pCommih->aSupBlk, nSupBlocks - 1); - (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); - pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; + blkIdx.tid = TABLE_TID(pTable); + blkIdx.uid = TABLE_UID(pTable); + blkIdx.hasLast = pBlock->last ? 1 : 0; + blkIdx.maxKey = pBlock->keyLast; + blkIdx.numOfBlocks = (uint32_t)nSupBlocks; + blkIdx.len = tlen; + blkIdx.offset = (uint32_t)offset; - if (newLast) { - (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); - pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info; + ASSERT(blkIdx.numOfBlocks > 0); + + if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +static int tsdbWriteBlockIdx(SCommitH *pCommih) { + SBlockIdx *pBlkIdx; + SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + size_t nidx = taosArrayGetSize(pCommih->aBlkIdx); + int tlen = 0, size; + int64_t offset; + + if (nidx <= 0) { + // All data are deleted + pHeadf->info.offset = 0; + pHeadf->info.len = 0; + return 0; + } + + for (size_t i = 0; i < nidx; i++) { + pBlkIdx = (SBlockIdx *)taosArrayGet(pCommih->aBlkIdx, i); + + size = tsdbEncodeSBlockIdx(NULL, pBlkIdx); + if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommih)), tlen + size) < 0) return -1; + + void *ptr = POINTER_SHIFT(TSDB_COMMIT_BUF(pCommih), tlen); + tsdbEncodeSBlockIdx(&ptr, pBlkIdx); + + tlen += size; + } + + tlen += sizeof(TSCKSUM); + if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommih)), tlen) < 0) return -1; + taosCalcChecksumAppend(0, (uint8_t *)TSDB_COMMIT_BUF(pCommih), tlen); + + if (tsdbAppendDFile(pHeadf, TSDB_COMMIT_BUF(pCommih), tlen, &offset) < tlen) { + tsdbError("vgId:%d failed to write block index part to file %s since %s", TSDB_COMMIT_REPO_ID(pCommih), + TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno)); + return -1; + } + + tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(TSDB_COMMIT_BUF(pCommih), tlen - sizeof(TSCKSUM))); + pHeadf->info.offset = (uint32_t)offset; + pHeadf->info.len = tlen; + + return 0; +} + +static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SMergeInfo mInfo; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + SDFile * pDFile; + bool isLast; + SBlock block; + + while (true) { + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0, + pCfg->update, &mInfo); + + if (pCommith->pDataCols->numOfRows <= 0) break; + + if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } else { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { + return -1; + } + } + + return 0; +} + +static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; + SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + TSKEY keyLimit; + int16_t colId = 0; + SMergeInfo mInfo; + SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; + SBlock block, supBlock; + SDFile * pDFile; + + if (bidx == nBlocks - 1) { + keyLimit = pCommith->maxKey; } else { - pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info; + keyLimit = pBlock[1].keyFirst - 1; } - pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info; + SSkipListIterator titer = *(pIter->pIter); + if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; + + tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, + pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); - pthread_rwlock_unlock(&(pFileH->fhlock)); + if (mInfo.nOperations == 0) { + // no new data to insert (all updates denied) + if (tsdbMoveBlock(pCommith, bidx) < 0) { + return -1; + } + *(pIter->pIter) = titer; + } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) { + // Ignore the block + ASSERT(0); + *(pIter->pIter) = titer; + } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) { + // Add a sub-block + tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols, + pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, + &mInfo); + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1; + + if (pBlock->numOfSubBlocks == 1) { + subBlocks[0] = *pBlock; + subBlocks[0].numOfSubBlocks = 0; + } else { + memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + sizeof(SBlock) * pBlock->numOfSubBlocks); + } + subBlocks[pBlock->numOfSubBlocks] = block; + supBlock = *pBlock; + supBlock.keyFirst = mInfo.keyFirst; + supBlock.keyLast = mInfo.keyLast; + supBlock.numOfSubBlocks++; + supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed; + supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock); + + if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1; + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1; + } return 0; +} -_err: - tfree(dataDir); - tsdbCloseHelperFile(pHelper, 1, pGroup); - return -1; +static int tsdbMoveBlock(SCommitH *pCommith, int bidx) { + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + SDFile *pDFile; + SBlock block; + bool isSameFile; + + ASSERT(pBlock->numOfSubBlocks > 0); + + if (pBlock->last) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isSameFile = pCommith->isLFileSame; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isSameFile = pCommith->isDFileSame; + } + + if (isSameFile) { + if (pBlock->numOfSubBlocks == 1) { + if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) { + return -1; + } + } else { + block = *pBlock; + block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk); + + if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset), + pBlock->numOfSubBlocks) < 0) { + return -1; + } + } + } else { + if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1; + if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; + } + + return 0; } -static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { - SMemTable *pMem = pRepo->imem; - STsdbMeta *pMeta = pRepo->tsdbMeta; +static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) { + if (taosArrayPush(pCommith->aSupBlk, pSupBlock) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } - SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); - if (iters == NULL) { + if (pSubBlocks && taosArrayPushBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; + return -1; } - if (tsdbRLockRepoMeta(pRepo) < 0) goto _err; + return 0; +} - // reference all tables - for (int i = 0; i < pMem->maxTables; i++) { - if (pMeta->tables[i] != NULL) { - tsdbRefTable(pMeta->tables[i]); - iters[i].pTable = pMeta->tables[i]; +static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlock block; + SDFile * pDFile; + bool isLast; + int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); + + int biter = 0; + while (true) { + tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows, + pCfg->update); + + if (pCommith->pDataCols->numOfRows == 0) break; + + if (isLastOneBlock) { + if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) { + pDFile = TSDB_COMMIT_LAST_FILE(pCommith); + isLast = true; + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; + } + } else { + pDFile = TSDB_COMMIT_DATA_FILE(pCommith); + isLast = false; } + + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; + if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1; } - if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err; + return 0; +} - for (int i = 0; i < pMem->maxTables; i++) { - if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) { - if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, + TSKEY maxKey, int maxRows, int8_t update) { + TSKEY key1 = INT64_MAX; + TSKEY key2 = INT64_MAX; + STSchema *pSchema = NULL; + + ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); + tdResetDataCols(pTarget); + + while (true) { + key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); + bool isRowDel = false; + SDataRow row = tsdbNextIterRow(pCommitIter->pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + key2 = INT64_MAX; + } else { + key2 = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } + + if (key1 == INT64_MAX && key2 == INT64_MAX) break; + + if (key1 < key2) { + for (int i = 0; i < pDataCols->numOfCols; i++) { + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); } - tSkipListIterNext(iters[i].pIter); + pTarget->numOfRows++; + (*iter)++; + } else if (key1 > key2) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendDataRowToDataCol(row, pSchema, pTarget); + } + + tSkipListIterNext(pCommitIter->pIter); + } else { + if (update) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendDataRowToDataCol(row, pSchema, pTarget); + } + } else { + ASSERT(!isRowDel); + + for (int i = 0; i < pDataCols->numOfCols; i++) { + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + pTarget->numOfRows++; + } + (*iter)++; + tSkipListIterNext(pCommitIter->pIter); } + + if (pTarget->numOfRows >= maxRows) break; } +} - return iters; +static void tsdbResetCommitFile(SCommitH *pCommith) { + pCommith->isRFileSet = false; + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + taosArrayClear(pCommith->aBlkIdx); +} -_err: - tsdbDestroyCommitIters(iters, pMem->maxTables); - return NULL; +static void tsdbResetCommitTable(SCommitH *pCommith) { + taosArrayClear(pCommith->aSubBlk); + taosArrayClear(pCommith->aSupBlk); + pCommith->pTable = NULL; +} + +static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { + SDiskID did; + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); + + tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + // Open read FSET + if (pSet) { + if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) { + return -1; + } + + pCommith->isRFileSet = true; + + if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is opened to read to commit", REPO_ID(pRepo), TSDB_FSET_FID(pSet), + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + } else { + pCommith->isRFileSet = false; + } + + // Set and open commit FSET + if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) { + // Create a new FSET to write data + tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo))); + + if (tsdbCreateDFileSet(pWSet, true) < 0) { + tsdbError("vgId:%d failed to create FSET %d at level %d disk id %d since %s", REPO_ID(pRepo), + TSDB_FSET_FID(pWSet), TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet), tstrerror(terrno)); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } + return -1; + } + + pCommith->isDFileSame = false; + pCommith->isLFileSame = false; + + tsdbDebug("vgId:%d FSET %d at level %d disk id %d is created to commit", REPO_ID(pRepo), TSDB_FSET_FID(pWSet), + TSDB_FSET_LEVEL(pWSet), TSDB_FSET_ID(pWSet)); + } else { + did.level = TSDB_FSET_LEVEL(pSet); + did.id = TSDB_FSET_ID(pSet); + + pCommith->wSet.fid = fid; + pCommith->wSet.state = 0; + + // TSDB_FILE_HEAD + SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith); + tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD); + if (tsdbCreateDFile(pWHeadf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWHeadf), + tstrerror(terrno)); + + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + + // TSDB_FILE_DATA + SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); + SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); + tsdbInitDFileEx(pWDataf, pRDataf); + if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + pCommith->isDFileSame = true; + + // TSDB_FILE_LAST + SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh)); + SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith); + if (pRLastf->info.size < 32 * 1024) { + tsdbInitDFileEx(pWLastf, pRLastf); + pCommith->isLFileSame = true; + + if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } else { + tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST); + pCommith->isLFileSame = false; + + if (tsdbCreateDFile(pWLastf, true) < 0) { + tsdbError("vgId:%d failed to create file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), + tstrerror(terrno)); + + tsdbCloseDFileSet(pWSet); + (void)tsdbRemoveDFile(pWHeadf); + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + return -1; + } + } + } + } + + return 0; } -static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) { - if (iters == NULL) return; +static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { + if (pCommith->isRFileSet) { + tsdbCloseAndUnsetFSet(&(pCommith->readh)); + } - for (int i = 1; i < maxTables; i++) { - if (iters[i].pTable != NULL) { - tsdbUnRefTable(iters[i].pTable); - tSkipListDestroyIter(iters[i].pIter); + if (!hasError) { + TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith)); + } + tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith)); +} + +static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { + STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; + + ASSERT(mergeRows > 0); + + if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) { + if (pBlock->last) { + if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true; + } else { + if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true; } } - free(iters); + return false; } + +static int tsdbApplyRtn(STsdbRepo *pRepo) { + SRtn rtn; + SFSIter fsiter; + STsdbFS * pfs = REPO_FS(pRepo); + SDFileSet *pSet; + + // Get retention snapshot + tsdbGetRtnSnap(pRepo, &rtn); + + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + while ((pSet = tsdbFSIterNext(&fsiter))) { + if (pSet->fid < rtn.minFid) { + tsdbInfo("vgId:%d FSET %d at level %d disk id %d expires, remove it", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet)); + continue; + } + + if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) { + return -1; + } + } + + return 0; +} + +static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) { + SDiskID did; + SDFileSet nSet; + STsdbFS * pfs = REPO_FS(pRepo); + int level; + + ASSERT(pSet->fid >= pRtn->minFid); + + level = tsdbGetFidLevel(pSet->fid, pRtn); + + tfsAllocDisk(level, &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + return -1; + } + + if (did.level > TSDB_FSET_LEVEL(pSet)) { + // Need to move the FSET to higher level + tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs)); + + if (tsdbCopyDFileSet(pSet, &nSet) < 0) { + tsdbError("vgId:%d failed to copy FSET %d from level %d to level %d since %s", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), did.level, tstrerror(terrno)); + return -1; + } + + if (tsdbUpdateDFileSet(pfs, &nSet) < 0) { + return -1; + } + + tsdbInfo("vgId:%d FSET %d is copied from level %d disk id %d to level %d disk id %d", REPO_ID(pRepo), pSet->fid, + TSDB_FSET_LEVEL(pSet), TSDB_FSET_ID(pSet), did.level, did.id); + } else { + // On a correct level + if (tsdbUpdateDFileSet(pfs, pSet) < 0) { + return -1; + } + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index 75a2cbcb8deb869922295e623144569e90a0cecf..9e8e4acd7ebea2209bf08798eb80f300a72927ab 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -13,11 +13,7 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tglobal.h" -#include "tlist.h" -#include "tref.h" -#include "tsdbMain.h" +#include "tsdbint.h" typedef struct { bool stop; @@ -35,7 +31,7 @@ typedef struct { static void *tsdbLoopCommit(void *arg); -SCommitQueue tsCommitQueue = {0}; +static SCommitQueue tsCommitQueue = {0}; int tsdbInitCommitQueue() { int nthreads = tsNumOfCommitThreads; diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c new file mode 100644 index 0000000000000000000000000000000000000000..c9f087a5cfe25967786b07c03217c89f1e1dd249 --- /dev/null +++ b/src/tsdb/src/tsdbFS.c @@ -0,0 +1,1244 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "tsdbint.h" +#include + +typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; +static const char *tsdbTxnFname[] = {"current.t", "current"}; +#define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) + +static int tsdbComparFidFSet(const void *arg1, const void *arg2); +static void tsdbResetFSStatus(SFSStatus *pStatus); +static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid); +static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo); +static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]); +static int tsdbOpenFSFromCurrent(STsdbRepo *pRepo); +static int tsdbScanAndTryFixFS(STsdbRepo *pRepo); +static int tsdbScanRootDir(STsdbRepo *pRepo); +static int tsdbScanDataDir(STsdbRepo *pRepo); +static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf); +static int tsdbRestoreCurrent(STsdbRepo *pRepo); +static int tsdbComparTFILE(const void *arg1, const void *arg2); +static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo); + +// ================== CURRENT file header info +static int tsdbEncodeFSHeader(void **buf, SFSHeader *pHeader) { + int tlen = 0; + + tlen += taosEncodeFixedU32(buf, pHeader->version); + tlen += taosEncodeFixedU32(buf, pHeader->len); + + return tlen; +} + +static void *tsdbDecodeFSHeader(void *buf, SFSHeader *pHeader) { + buf = taosDecodeFixedU32(buf, &(pHeader->version)); + buf = taosDecodeFixedU32(buf, &(pHeader->len)); + + return buf; +} + +// ================== STsdbFSMeta +static int tsdbEncodeFSMeta(void **buf, STsdbFSMeta *pMeta) { + int tlen = 0; + + tlen += taosEncodeFixedU32(buf, pMeta->version); + tlen += taosEncodeFixedI64(buf, pMeta->totalPoints); + tlen += taosEncodeFixedI64(buf, pMeta->totalStorage); + + return tlen; +} + +static void *tsdbDecodeFSMeta(void *buf, STsdbFSMeta *pMeta) { + buf = taosDecodeFixedU32(buf, &(pMeta->version)); + buf = taosDecodeFixedI64(buf, &(pMeta->totalPoints)); + buf = taosDecodeFixedI64(buf, &(pMeta->totalStorage)); + + return buf; +} + +// ================== SFSStatus +static int tsdbEncodeDFileSetArray(void **buf, SArray *pArray) { + int tlen = 0; + uint64_t nset = taosArrayGetSize(pArray); + + tlen += taosEncodeFixedU64(buf, nset); + for (size_t i = 0; i < nset; i++) { + SDFileSet *pSet = taosArrayGet(pArray, i); + + tlen += tsdbEncodeDFileSet(buf, pSet); + } + + return tlen; +} + +static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) { + uint64_t nset; + SDFileSet dset; + + taosArrayClear(pArray); + + buf = taosDecodeFixedU64(buf, &nset); + for (size_t i = 0; i < nset; i++) { + buf = tsdbDecodeDFileSet(buf, &dset); + taosArrayPush(pArray, (void *)(&dset)); + } + return buf; +} + +static int tsdbEncodeFSStatus(void **buf, SFSStatus *pStatus) { + ASSERT(pStatus->pmf); + + int tlen = 0; + + tlen += tsdbEncodeSMFile(buf, pStatus->pmf); + tlen += tsdbEncodeDFileSetArray(buf, pStatus->df); + + return tlen; +} + +static void *tsdbDecodeFSStatus(void *buf, SFSStatus *pStatus) { + tsdbResetFSStatus(pStatus); + + pStatus->pmf = &(pStatus->mf); + + buf = tsdbDecodeSMFile(buf, pStatus->pmf); + buf = tsdbDecodeDFileSetArray(buf, pStatus->df); + + return buf; +} + +static SFSStatus *tsdbNewFSStatus(int maxFSet) { + SFSStatus *pStatus = (SFSStatus *)calloc(1, sizeof(*pStatus)); + if (pStatus == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + TSDB_FILE_SET_CLOSED(&(pStatus->mf)); + + pStatus->df = taosArrayInit(maxFSet, sizeof(SDFileSet)); + if (pStatus->df == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + free(pStatus); + return NULL; + } + + return pStatus; +} + +static SFSStatus *tsdbFreeFSStatus(SFSStatus *pStatus) { + if (pStatus) { + pStatus->df = taosArrayDestroy(pStatus->df); + free(pStatus); + } + + return NULL; +} + +static void tsdbResetFSStatus(SFSStatus *pStatus) { + if (pStatus == NULL) { + return; + } + + TSDB_FILE_SET_CLOSED(&(pStatus->mf)); + + pStatus->pmf = NULL; + taosArrayClear(pStatus->df); +} + +static void tsdbSetStatusMFile(SFSStatus *pStatus, const SMFile *pMFile) { + ASSERT(pStatus->pmf == NULL); + + pStatus->pmf = &(pStatus->mf); + tsdbInitMFileEx(pStatus->pmf, (SMFile *)pMFile); +} + +static int tsdbAddDFileSetToStatus(SFSStatus *pStatus, const SDFileSet *pSet) { + if (taosArrayPush(pStatus->df, (void *)pSet) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + TSDB_FSET_SET_CLOSED(((SDFileSet *)taosArrayGetLast(pStatus->df))); + + return 0; +} + +// ================== STsdbFS +STsdbFS *tsdbNewFS(STsdbCfg *pCfg) { + int keep = pCfg->keep; + int days = pCfg->daysPerFile; + int maxFSet = TSDB_MAX_FSETS(keep, days); + STsdbFS *pfs; + + pfs = (STsdbFS *)calloc(1, sizeof(*pfs)); + if (pfs == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + int code = pthread_rwlock_init(&(pfs->lock), NULL); + if (code) { + terrno = TAOS_SYSTEM_ERROR(code); + free(pfs); + return NULL; + } + + pfs->cstatus = tsdbNewFSStatus(maxFSet); + if (pfs->cstatus == NULL) { + tsdbFreeFS(pfs); + return NULL; + } + + pfs->metaCache = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pfs->metaCache == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbFreeFS(pfs); + return NULL; + } + + pfs->nstatus = tsdbNewFSStatus(maxFSet); + if (pfs->nstatus == NULL) { + tsdbFreeFS(pfs); + return NULL; + } + + return pfs; +} + +void *tsdbFreeFS(STsdbFS *pfs) { + if (pfs) { + pfs->nstatus = tsdbFreeFSStatus(pfs->nstatus); + taosHashCleanup(pfs->metaCache); + pfs->metaCache = NULL; + pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus); + pthread_rwlock_destroy(&(pfs->lock)); + free(pfs); + } + + return NULL; +} + +int tsdbOpenFS(STsdbRepo *pRepo) { + STsdbFS *pfs = REPO_FS(pRepo); + char current[TSDB_FILENAME_LEN] = "\0"; + + ASSERT(pfs != NULL); + + tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + + if (access(current, F_OK) == 0) { + if (tsdbOpenFSFromCurrent(pRepo) < 0) { + tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbScanAndTryFixDFilesHeader(pRepo); + } else { + if (tsdbRestoreCurrent(pRepo) < 0) { + tsdbError("vgId:%d failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + if (tsdbScanAndTryFixFS(pRepo) < 0) { + tsdbError("vgId:%d failed to scan and fix FS since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + // Load meta cache if has meta file + if ((!(pRepo->state & TSDB_STATE_BAD_META)) && tsdbLoadMetaCache(pRepo, true) < 0) { + tsdbError("vgId:%d failed to open FS while loading meta cache since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + return 0; +} + +void tsdbCloseFS(STsdbRepo *pRepo) { + // Do nothing +} + +// Start a new transaction to modify the file system +void tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd) { + STsdbFS *pfs = REPO_FS(pRepo); + ASSERT(pfs->intxn == false); + + pfs->intxn = true; + tsdbResetFSStatus(pfs->nstatus); + pfs->nstatus->meta = pfs->cstatus->meta; + if (pfs->cstatus->pmf == NULL) { + pfs->nstatus->meta.version = 0; + } else { + pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1; + } + pfs->nstatus->meta.totalPoints = pfs->cstatus->meta.totalPoints + pointsAdd; + pfs->nstatus->meta.totalStorage = pfs->cstatus->meta.totalStorage += storageAdd; +} + +void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta) { pfs->nstatus->meta = *pMeta; } + +int tsdbEndFSTxn(STsdbRepo *pRepo) { + STsdbFS *pfs = REPO_FS(pRepo); + ASSERT(FS_IN_TXN(pfs)); + SFSStatus *pStatus; + + // Write current file system snapshot + if (tsdbSaveFSStatus(pfs->nstatus, REPO_ID(pRepo)) < 0) { + tsdbEndFSTxnWithError(pfs); + return -1; + } + + // Make new + tsdbWLockFS(pfs); + pStatus = pfs->cstatus; + pfs->cstatus = pfs->nstatus; + pfs->nstatus = pStatus; + tsdbUnLockFS(pfs); + + // Apply actual change to each file and SDFileSet + tsdbApplyFSTxnOnDisk(pfs->nstatus, pfs->cstatus); + + pfs->intxn = false; + return 0; +} + +int tsdbEndFSTxnWithError(STsdbFS *pfs) { + tsdbApplyFSTxnOnDisk(pfs->nstatus, pfs->cstatus); + // TODO: if mf change, reload pfs->metaCache + pfs->intxn = false; + return 0; +} + +void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile) { tsdbSetStatusMFile(pfs->nstatus, pMFile); } + +int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); } + +static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) { + SFSHeader fsheader; + void * pBuf = NULL; + void * ptr; + char hbuf[TSDB_FILE_HEAD_SIZE] = "\0"; + char tfname[TSDB_FILENAME_LEN] = "\0"; + char cfname[TSDB_FILENAME_LEN] = "\0"; + + tsdbGetTxnFname(vid, TSDB_TXN_TEMP_FILE, tfname); + tsdbGetTxnFname(vid, TSDB_TXN_CURR_FILE, cfname); + + int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + fsheader.version = TSDB_FS_VERSION; + if (pStatus->pmf == NULL) { + ASSERT(taosArrayGetSize(pStatus->df) == 0); + fsheader.len = 0; + } else { + fsheader.len = tsdbEncodeFSStatus(NULL, pStatus) + sizeof(TSCKSUM); + } + + // Encode header part and write + ptr = hbuf; + tsdbEncodeFSHeader(&ptr, &fsheader); + tsdbEncodeFSMeta(&ptr, &(pStatus->meta)); + + taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE); + + if (taosWrite(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + terrno = TAOS_SYSTEM_ERROR(errno); + close(fd); + remove(tfname); + return -1; + } + + // Encode file status and write to file + if (fsheader.len > 0) { + if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { + close(fd); + remove(tfname); + return -1; + } + + ptr = pBuf; + tsdbEncodeFSStatus(&ptr, pStatus); + taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len); + + if (taosWrite(fd, pBuf, fsheader.len) < fsheader.len) { + terrno = TAOS_SYSTEM_ERROR(errno); + close(fd); + (void)remove(tfname); + taosTZfree(pBuf); + return -1; + } + } + + // fsync, close and rename + if (fsync(fd) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + close(fd); + remove(tfname); + taosTZfree(pBuf); + return -1; + } + + (void)close(fd); + (void)taosRename(tfname, cfname); + taosTZfree(pBuf); + + return 0; +} + +static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo) { + int ifrom = 0; + int ito = 0; + size_t sizeFrom, sizeTo; + SDFileSet *pSetFrom; + SDFileSet *pSetTo; + + sizeFrom = taosArrayGetSize(pFrom->df); + sizeTo = taosArrayGetSize(pTo->df); + + // Apply meta file change + (void)tsdbApplyMFileChange(pFrom->pmf, pTo->pmf); + + // Apply SDFileSet change + if (ifrom >= sizeFrom) { + pSetFrom = NULL; + } else { + pSetFrom = taosArrayGet(pFrom->df, ifrom); + } + + if (ito >= sizeTo) { + pSetTo = NULL; + } else { + pSetTo = taosArrayGet(pTo->df, ito); + } + + while (true) { + if ((pSetTo == NULL) && (pSetFrom == NULL)) break; + + if (pSetTo == NULL || (pSetFrom && pSetFrom->fid < pSetTo->fid)) { + tsdbApplyDFileSetChange(pSetFrom, NULL); + + ifrom++; + if (ifrom >= sizeFrom) { + pSetFrom = NULL; + } else { + pSetFrom = taosArrayGet(pFrom->df, ifrom); + } + } else if (pSetFrom == NULL || pSetFrom->fid > pSetTo->fid) { + // Do nothing + ito++; + if (ito >= sizeTo) { + pSetTo = NULL; + } else { + pSetTo = taosArrayGet(pTo->df, ito); + } + } else { + tsdbApplyDFileSetChange(pSetFrom, pSetTo); + + ifrom++; + if (ifrom >= sizeFrom) { + pSetFrom = NULL; + } else { + pSetFrom = taosArrayGet(pFrom->df, ifrom); + } + + ito++; + if (ito >= sizeTo) { + pSetTo = NULL; + } else { + pSetTo = taosArrayGet(pTo->df, ito); + } + } + } +} + +// ================== SFSIter +// ASSUMPTIONS: the FS Should be read locked when calling these functions +void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction) { + pIter->pfs = pfs; + pIter->direction = direction; + + size_t size = taosArrayGetSize(pfs->cstatus->df); + + pIter->version = pfs->cstatus->meta.version; + + if (size == 0) { + pIter->index = -1; + pIter->fid = TSDB_IVLD_FID; + } else { + if (direction == TSDB_FS_ITER_FORWARD) { + pIter->index = 0; + } else { + pIter->index = (int)(size - 1); + } + + pIter->fid = ((SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index))->fid; + } +} + +void tsdbFSIterSeek(SFSIter *pIter, int fid) { + STsdbFS *pfs = pIter->pfs; + size_t size = taosArrayGetSize(pfs->cstatus->df); + + int flags; + if (pIter->direction == TSDB_FS_ITER_FORWARD) { + flags = TD_GE; + } else { + flags = TD_LE; + } + + void *ptr = taosbsearch(&fid, pfs->cstatus->df->pData, size, sizeof(SDFileSet), tsdbComparFidFSet, flags); + if (ptr == NULL) { + pIter->index = -1; + pIter->fid = TSDB_IVLD_FID; + } else { + pIter->index = (int)(TARRAY_ELEM_IDX(pfs->cstatus->df, ptr)); + pIter->fid = ((SDFileSet *)ptr)->fid; + } +} + +SDFileSet *tsdbFSIterNext(SFSIter *pIter) { + STsdbFS * pfs = pIter->pfs; + SDFileSet *pSet; + + if (pIter->index < 0) { + ASSERT(pIter->fid == TSDB_IVLD_FID); + return NULL; + } + + ASSERT(pIter->fid != TSDB_IVLD_FID); + + if (pIter->version != pfs->cstatus->meta.version) { + pIter->version = pfs->cstatus->meta.version; + tsdbFSIterSeek(pIter, pIter->fid); + } + + if (pIter->index < 0) { + return NULL; + } + + pSet = (SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index); + ASSERT(pSet->fid == pIter->fid); + + if (pIter->direction == TSDB_FS_ITER_FORWARD) { + pIter->index++; + if (pIter->index >= taosArrayGetSize(pfs->cstatus->df)) { + pIter->index = -1; + } + } else { + pIter->index--; + } + + if (pIter->index >= 0) { + pIter->fid = ((SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index))->fid; + } else { + pIter->fid = TSDB_IVLD_FID; + } + + return pSet; +} + +static int tsdbComparFidFSet(const void *arg1, const void *arg2) { + int fid = *(int *)arg1; + SDFileSet *pSet = (SDFileSet *)arg2; + + if (fid < pSet->fid) { + return -1; + } else if (fid == pSet->fid) { + return 0; + } else { + return 1; + } +} + +static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]) { + snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", TFS_PRIMARY_PATH(), repoid, tsdbTxnFname[ftype]); +} + +static int tsdbOpenFSFromCurrent(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + int fd = -1; + void * buffer = NULL; + SFSHeader fsheader; + char current[TSDB_FILENAME_LEN] = "\0"; + void * ptr; + + tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current); + + // current file exists, try to recover + fd = open(current, O_RDONLY | O_BINARY); + if (fd < 0) { + tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (tsdbMakeRoom(&buffer, TSDB_FILE_HEAD_SIZE) < 0) { + goto _err; + } + + int nread = (int)taosRead(fd, buffer, TSDB_FILE_HEAD_SIZE); + if (nread < 0) { + tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (nread < TSDB_FILE_HEAD_SIZE) { + tsdbError("vgId:%d failed to read header of file %s, read bytes:%d", REPO_ID(pRepo), current, nread); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + if (!taosCheckChecksumWhole((uint8_t *)buffer, TSDB_FILE_HEAD_SIZE)) { + tsdbError("vgId:%d header of file %s failed checksum check", REPO_ID(pRepo), current); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + SFSStatus *pStatus = pfs->cstatus; + ptr = buffer; + ptr = tsdbDecodeFSHeader(ptr, &fsheader); + ptr = tsdbDecodeFSMeta(ptr, &(pStatus->meta)); + + if (fsheader.version != TSDB_FS_VERSION) { + // TODO: handle file version change + } + + if (fsheader.len > 0) { + if (tsdbMakeRoom(&buffer, fsheader.len) < 0) { + goto _err; + } + + nread = (int)taosRead(fd, buffer, fsheader.len); + if (nread < 0) { + tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (nread < fsheader.len) { + tsdbError("vgId:%d failed to read %d bytes from file %s", REPO_ID(pRepo), fsheader.len, current); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + if (!taosCheckChecksumWhole((uint8_t *)buffer, fsheader.len)) { + tsdbError("vgId:%d file %s is corrupted since wrong checksum", REPO_ID(pRepo), current); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + ptr = buffer; + ptr = tsdbDecodeFSStatus(ptr, pStatus); + } else { + tsdbResetFSStatus(pStatus); + } + + taosTZfree(buffer); + close(fd); + + return 0; + +_err: + if (fd >= 0) { + close(fd); + } + taosTZfree(buffer); + return -1; +} + +// Scan and try to fix incorrect files +static int tsdbScanAndTryFixFS(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SFSStatus *pStatus = pfs->cstatus; + + if (tsdbScanAndTryFixMFile(pRepo) < 0) { + tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + size_t size = taosArrayGetSize(pStatus->df); + + for (size_t i = 0; i < size; i++) { + SDFileSet *pSet = (SDFileSet *)taosArrayGet(pStatus->df, i); + + if (tsdbScanAndTryFixDFileSet(pRepo, pSet) < 0) { + tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + // remove those unused files + tsdbScanRootDir(pRepo); + tsdbScanDataDir(pRepo); + return 0; +} + +int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) { + char tbuf[128]; + STsdbFS * pfs = REPO_FS(pRepo); + SMFile mf; + SMFile * pMFile = &mf; + void * pBuf = NULL; + SKVRecord rInfo; + int64_t maxBufSize = 0; + SMFInfo minfo; + + taosHashEmpty(pfs->metaCache); + + // No meta file, just return + if (pfs->cstatus->pmf == NULL) return 0; + + mf = pfs->cstatus->mf; + // Load cache first + if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { + return -1; + } + + if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) { + tsdbCloseMFile(pMFile); + return -1; + } + + while (true) { + int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord)); + if (tsize == 0) break; + + if (tsize < 0) { + tsdbError("vgId:%d failed to read META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (tsize < sizeof(SKVRecord)) { + tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord), + TSDB_FILE_FULL_NAME(pMFile)); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbCloseMFile(pMFile); + return -1; + } + + void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo); + ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord)); + // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true); + + if (rInfo.offset < 0) { + taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid)); +#if 0 + pStore->info.size += sizeof(SKVRecord); + pStore->info.nRecords--; + pStore->info.nDels++; + pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); +#endif + } else { + ASSERT(rInfo.offset > 0 && rInfo.size > 0); + if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) { + tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pMFile)); + terrno = TSDB_CODE_COM_OUT_OF_MEMORY; + tsdbCloseMFile(pMFile); + return -1; + } + + maxBufSize = MAX(maxBufSize, rInfo.size); + + if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + tsdbCloseMFile(pMFile); + return -1; + } + +#if 0 + pStore->info.size += (sizeof(SKVRecord) + rInfo.size); + pStore->info.nRecords++; +#endif + } + } + + if (recoverMeta) { + pBuf = malloc((size_t)maxBufSize); + if (pBuf == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCloseMFile(pMFile); + return -1; + } + + SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL); + while (pRecord) { + if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { + tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + tfree(pBuf); + tsdbCloseMFile(pMFile); + return -1; + } + + int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); + if (nread < 0) { + tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + tfree(pBuf); + tsdbCloseMFile(pMFile); + return -1; + } + + if (nread < pRecord->size) { + tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tfree(pBuf); + tsdbCloseMFile(pMFile); + return -1; + } + + if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) { + tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid, + tstrerror(terrno)); + tfree(pBuf); + tsdbCloseMFile(pMFile); + return -1; + } + + pRecord = taosHashIterate(pfs->metaCache, pRecord); + } + + tsdbOrgMeta(pRepo); + } + + tsdbCloseMFile(pMFile); + tfree(pBuf); + return 0; +} + +static int tsdbScanRootDir(STsdbRepo *pRepo) { + char rootDir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + STsdbFS * pfs = REPO_FS(pRepo); + const TFILE *pf; + + tsdbGetRootDir(REPO_ID(pRepo), rootDir); + TDIR *tdir = tfsOpendir(rootDir); + if (tdir == NULL) { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); + return -1; + } + + while ((pf = tfsReaddir(tdir))) { + tfsbasename(pf, bname); + + if (strcmp(bname, tsdbTxnFname[TSDB_TXN_CURR_FILE]) == 0 || strcmp(bname, "data") == 0) { + // Skip current file and data directory + continue; + } + + if (pfs->cstatus->pmf && tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) { + continue; + } + + (void)tfsremove(pf); + tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); + } + + tfsClosedir(tdir); + + return 0; +} + +static int tsdbScanDataDir(STsdbRepo *pRepo) { + char dataDir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + STsdbFS * pfs = REPO_FS(pRepo); + const TFILE *pf; + + tsdbGetDataDir(REPO_ID(pRepo), dataDir); + TDIR *tdir = tfsOpendir(dataDir); + if (tdir == NULL) { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno)); + return -1; + } + + while ((pf = tfsReaddir(tdir))) { + tfsbasename(pf, bname); + + if (!tsdbIsTFileInFS(pfs, pf)) { + (void)tfsremove(pf); + tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf)); + } + } + + tfsClosedir(tdir); + + return 0; +} + +static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) { + SFSIter fsiter; + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + SDFileSet *pSet; + + while ((pSet = tsdbFSIterNext(&fsiter))) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); + if (tfsIsSameFile(pf, TSDB_FILE_F(pDFile))) { + return true; + } + } + } + + return false; +} + +static int tsdbRestoreMeta(STsdbRepo *pRepo) { + char rootDir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + TDIR * tdir = NULL; + const TFILE *pf = NULL; + const char * pattern = "^meta(-ver[0-9]+)?$"; + regex_t regex; + STsdbFS * pfs = REPO_FS(pRepo); + + regcomp(®ex, pattern, REG_EXTENDED); + + tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo)); + + tsdbGetRootDir(REPO_ID(pRepo), rootDir); + + tdir = tfsOpendir(rootDir); + if (tdir == NULL) { + tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno)); + regfree(®ex); + return -1; + } + + while ((pf = tfsReaddir(tdir))) { + tfsbasename(pf, bname); + + if (strcmp(bname, "data") == 0) { + // Skip the data/ directory + continue; + } + + if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) { + // Skip current.t file + tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); + (void)tfsremove(pf); + continue; + } + + int code = regexec(®ex, bname, 0, NULL, 0); + if (code == 0) { + // Match + if (pfs->cstatus->pmf != NULL) { + tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), TFILE_NAME(pf)); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tfsClosedir(tdir); + regfree(®ex); + return -1; + } else { + uint32_t version = 0; + if (strcmp(bname, "meta") != 0) { + sscanf(bname, "meta-ver%" PRIu32, &version); + pfs->cstatus->meta.version = version; + } + + pfs->cstatus->pmf = &(pfs->cstatus->mf); + pfs->cstatus->pmf->f = *pf; + TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf); + + if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) { + tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + tfsClosedir(tdir); + regfree(®ex); + return -1; + } + + if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) { + tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbCloseMFile(pfs->cstatus->pmf); + tfsClosedir(tdir); + regfree(®ex); + return -1; + } + + tsdbCloseMFile(pfs->cstatus->pmf); + } + } else if (code == REG_NOMATCH) { + // Not match + tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); + tfsremove(pf); + continue; + } else { + // Has other error + tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + tfsClosedir(tdir); + regfree(®ex); + return -1; + } + } + + if (pfs->cstatus->pmf) { + tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf)); + } else { + tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo)); + } + + tfsClosedir(tdir); + regfree(®ex); + return 0; +} + +static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { + char dataDir[TSDB_FILENAME_LEN]; + char bname[TSDB_FILENAME_LEN]; + TDIR * tdir = NULL; + const TFILE *pf = NULL; + const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last)(-ver[0-9]+)?$"; + SArray * fArray = NULL; + regex_t regex; + STsdbFS * pfs = REPO_FS(pRepo); + + tsdbGetDataDir(REPO_ID(pRepo), dataDir); + + // Resource allocation and init + regcomp(®ex, pattern, REG_EXTENDED); + + fArray = taosArrayInit(1024, sizeof(TFILE)); + if (fArray == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, + tstrerror(terrno)); + regfree(®ex); + return -1; + } + + tdir = tfsOpendir(dataDir); + if (tdir == NULL) { + tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir, + tstrerror(terrno)); + taosArrayDestroy(fArray); + regfree(®ex); + return -1; + } + + while ((pf = tfsReaddir(tdir))) { + tfsbasename(pf, bname); + + int code = regexec(®ex, bname, 0, NULL, 0); + if (code == 0) { + if (taosArrayPush(fArray, (void *)pf) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tfsClosedir(tdir); + taosArrayDestroy(fArray); + regfree(®ex); + return -1; + } + } else if (code == REG_NOMATCH) { + // Not match + tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf)); + (void)tfsremove(pf); + continue; + } else { + // Has other error + tsdbError("vgId:%d failed to restore DFileSet Array while run regexec since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + tfsClosedir(tdir); + taosArrayDestroy(fArray); + regfree(®ex); + return -1; + } + } + + tfsClosedir(tdir); + regfree(®ex); + + // Sort the array according to file name + taosArraySort(fArray, tsdbComparTFILE); + + size_t index = 0; + // Loop to recover each file set + for (;;) { + if (index >= taosArrayGetSize(fArray)) { + break; + } + + SDFileSet fset = {0}; + + TSDB_FSET_SET_CLOSED(&fset); + + // Loop to recover ONE fset + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); + + if (index >= taosArrayGetSize(fArray)) { + tsdbError("vgId:%d incomplete DFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + taosArrayDestroy(fArray); + return -1; + } + + pf = taosArrayGet(fArray, index); + + int tvid, tfid; + TSDB_FILE_T ttype; + uint32_t tversion; + char bname[TSDB_FILENAME_LEN]; + + tfsbasename(pf, bname); + tsdbParseDFilename(bname, &tvid, &tfid, &ttype, &tversion); + + ASSERT(tvid == REPO_ID(pRepo)); + + if (ftype == 0) { + fset.fid = tfid; + } else { + if (tfid != fset.fid) { + tsdbError("vgId:%d incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + taosArrayDestroy(fArray); + return -1; + } + } + + if (ttype != ftype) { + tsdbError("vgId:%d incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid); + taosArrayDestroy(fArray); + return -1; + } + + pDFile->f = *pf; + + if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { + tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); + taosArrayDestroy(fArray); + return -1; + } + + if (tsdbLoadDFileHeader(pDFile, &(pDFile->info)) < 0) { + tsdbError("vgId:%d failed to load DFile %s header since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + tstrerror(terrno)); + taosArrayDestroy(fArray); + return -1; + } + + tsdbCloseDFile(pDFile); + index++; + } + + tsdbInfo("vgId:%d FSET %d is restored", REPO_ID(pRepo), fset.fid); + taosArrayPush(pfs->cstatus->df, &fset); + } + + // Resource release + taosArrayDestroy(fArray); + + return 0; +} + +static int tsdbRestoreCurrent(STsdbRepo *pRepo) { + // Loop to recover mfile + if (tsdbRestoreMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + // Loop to recover dfile set + if (tsdbRestoreDFileSet(pRepo) < 0) { + tsdbError("vgId:%d failed to restore DFileSet since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (tsdbSaveFSStatus(pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) { + tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + return 0; +} + +static int tsdbComparTFILE(const void *arg1, const void *arg2) { + TFILE *pf1 = (TFILE *)arg1; + TFILE *pf2 = (TFILE *)arg2; + + int vid1, fid1, vid2, fid2; + TSDB_FILE_T ftype1, ftype2; + uint32_t version1, version2; + char bname1[TSDB_FILENAME_LEN]; + char bname2[TSDB_FILENAME_LEN]; + + tfsbasename(pf1, bname1); + tfsbasename(pf2, bname2); + tsdbParseDFilename(bname1, &vid1, &fid1, &ftype1, &version1); + tsdbParseDFilename(bname2, &vid2, &fid2, &ftype2, &version2); + + if (fid1 < fid2) { + return -1; + } else if (fid1 > fid2) { + return 1; + } else { + if (ftype1 < ftype2) { + return -1; + } else if (ftype1 > ftype2) { + return 1; + } else { + return 0; + } + } +} + +static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SFSStatus *pStatus = pfs->cstatus; + SDFInfo info; + + for (size_t i = 0; i < taosArrayGetSize(pStatus->df); i++) { + SDFileSet fset; + tsdbInitDFileSetEx(&fset, (SDFileSet *)taosArrayGet(pStatus->df, i)); + + tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); + + if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); + continue; + } + + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); + + if ((tsdbLoadDFileHeader(pDFile, &info) < 0) || pDFile->info.size != info.size || + pDFile->info.magic != info.magic) { + if (tsdbUpdateDFileHeader(pDFile) < 0) { + tsdbError("vgId:%d failed to update DFile header of %s since %s, continue", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); + } else { + tsdbInfo("vgId:%d DFile header of %s is updated", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + TSDB_FILE_FSYNC(pDFile); + } + } else { + tsdbDebug("vgId:%d DFile header of %s is correct", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + } + } + + tsdbCloseDFileSet(&fset); + } +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 5d8933d14142e4c569808fd932efbed63a9ed8e9..5db993e46346dcbde81507faa774d431a1607781 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -12,433 +12,482 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define _DEFAULT_SOURCE -#define TAOS_RANDOM_FILE_FAIL_TEST -#include -#include "os.h" -#include "talgo.h" -#include "tchecksum.h" -#include "tsdbMain.h" -#include "tutil.h" +#include "tsdbint.h" -const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"}; +static const char *TSDB_FNAME_SUFFIX[] = { + "head", // TSDB_FILE_HEAD + "data", // TSDB_FILE_DATA + "last", // TSDB_FILE_LAST + "", // TSDB_FILE_MAX + "meta" // TSDB_FILE_META +}; -static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); -static void tsdbDestroyFile(SFile *pFile); -static int compFGroup(const void *arg1, const void *arg2); -static int keyFGroupCompFunc(const void *key, const void *fgroup); -static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo); -static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep); -static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days); +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); +static int tsdbRollBackMFile(SMFile *pMFile); +static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo); +static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); +static int tsdbRollBackDFile(SDFile *pDFile); -// ---------------- INTERNAL FUNCTIONS ---------------- -STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { - STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(*pFileH)); - if (pFileH == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } +// ============== SMFile +void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { + char fname[TSDB_FILENAME_LEN]; - int code = pthread_rwlock_init(&(pFileH->fhlock), NULL); - if (code != 0) { - tsdbError("vgId:%d failed to init file handle lock since %s", pCfg->tsdbId, strerror(code)); - terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK); - pFileH->maxFGroups = TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile); + memset(&(pMFile->info), 0, sizeof(pMFile->info)); + pMFile->info.magic = TSDB_FILE_INIT_MAGIC; - pFileH->pFGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup)); - if (pFileH->pFGroup == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } + tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname); + tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname); +} + +void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) { + *pMFile = *pOMFile; + TSDB_FILE_SET_CLOSED(pMFile); +} + +int tsdbEncodeSMFile(void **buf, SMFile *pMFile) { + int tlen = 0; - return pFileH; + tlen += tsdbEncodeMFInfo(buf, &(pMFile->info)); + tlen += tfsEncodeFile(buf, &(pMFile->f)); -_err: - tsdbFreeFileH(pFileH); - return NULL; + return tlen; } -void tsdbFreeFileH(STsdbFileH *pFileH) { - if (pFileH) { - pthread_rwlock_destroy(&pFileH->fhlock); - tfree(pFileH->pFGroup); - free(pFileH); - } +void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) { + buf = tsdbDecodeMFInfo(buf, &(pMFile->info)); + buf = tfsDecodeFile(buf, &(pMFile->f)); + TSDB_FILE_SET_CLOSED(pMFile); + + return buf; +} + +int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) { + int tlen = 0; + + tlen += tsdbEncodeMFInfo(buf, &(pMFile->info)); + tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile)); + + return tlen; } -int tsdbOpenFileH(STsdbRepo *pRepo) { - ASSERT(pRepo != NULL && pRepo->tsdbFileH != NULL); +void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) { + char *aname; + buf = tsdbDecodeMFInfo(buf, &(pMFile->info)); + buf = taosDecodeString(buf, &aname); + strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN); + TSDB_FILE_SET_CLOSED(pMFile); + + tfree(aname); - char * tDataDir = NULL; - DIR * dir = NULL; - int fid = 0; - int vid = 0; - regex_t regex1 = {0}, regex2 = {0}; - int code = 0; - char fname[TSDB_FILENAME_LEN] = "\0"; + return buf; +} - SFileGroup fileGroup = {0}; - STsdbFileH *pFileH = pRepo->tsdbFileH; - STsdbCfg * pCfg = &(pRepo->config); +int tsdbApplyMFileChange(SMFile *from, SMFile *to) { + if (from == NULL && to == NULL) return 0; - tDataDir = tsdbGetDataDirName(pRepo->rootDir); - if (tDataDir == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + if (from != NULL) { + if (to == NULL) { + return tsdbRemoveMFile(from); + } else { + if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) { + if (from->info.size > to->info.size) { + tsdbRollBackMFile(to); + } + } else { + return tsdbRemoveMFile(from); + } + } } - dir = opendir(tDataDir); - if (dir == NULL) { - if (errno == ENOENT) { - tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir); - terrno = TAOS_SYSTEM_ERROR(errno); + return 0; +} - if (taosMkDir(tDataDir, 0755) < 0) { - tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; +int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) { + ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC); + + pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (pMFile->fd < 0) { + if (errno == ENOENT) { + // Try to create directory recursively + char *s = strdup(TFILE_REL_NAME(&(pMFile->f))); + if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) { + tfree(s); + return -1; } + tfree(s); - dir = opendir(tDataDir); - if (dir == NULL) { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (pMFile->fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + return -1; } } else { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + return -1; } } - code = regcomp(®ex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED); - if (code != 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + if (!updateHeader) { + return 0; } - code = regcomp(®ex2, "^v[0-9]+f[0-9]+\\.(h|d|l|s)$", REG_EXTENDED); - if (code != 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + pMFile->info.size += TSDB_FILE_HEAD_SIZE; + + if (tsdbUpdateMFileHeader(pMFile) < 0) { + tsdbCloseMFile(pMFile); + tsdbRemoveMFile(pMFile); + return -1; } - int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); + return 0; +} - struct dirent *dp = NULL; - while ((dp = readdir(dir)) != NULL) { - if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue; +int tsdbUpdateMFileHeader(SMFile *pMFile) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - code = regexec(®ex1, dp->d_name, 0, NULL, 0); - if (code == 0) { - sscanf(dp->d_name, "v%df%d", &vid, &fid); - if (vid != REPO_ID(pRepo)) { - tsdbError("vgId:%d invalid file %s exists, ignore it", REPO_ID(pRepo), dp->d_name); - continue; - } + if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) { + return -1; + } - if (fid < mfid) { - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - tsdbGetDataFileName(pRepo->rootDir, pCfg->tsdbId, fid, type, fname); - (void)remove(fname); - } - continue; - } + void *ptr = buf; + tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile)); - if (tsdbSearchFGroup(pFileH, fid, TD_EQ) != NULL) continue; - memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); - fileGroup.fileId = fid; - - tsdbInitFileGroup(&fileGroup, pRepo); - } else if (code == REG_NOMATCH) { - code = regexec(®ex2, dp->d_name, 0, NULL, 0); - if (code == 0) { - size_t tsize = strlen(tDataDir) + strlen(dp->d_name) + 2; - char * fname1 = malloc(tsize); - if (fname1 == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - sprintf(fname1, "%s/%s", tDataDir, dp->d_name); - - tsize = tsize + 64; - char *fname2 = malloc(tsize); - if (fname2 == NULL) { - free(fname1); - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - sprintf(fname2, "%s/%s_back_%" PRId64, tDataDir, dp->d_name, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI)); + taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { + return -1; + } - (void)rename(fname1, fname2); + return 0; +} - tsdbDebug("vgId:%d file %s exists, backup it as %s", REPO_ID(pRepo), fname1, fname2); +int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - free(fname1); - free(fname2); - continue; - } else if (code == REG_NOMATCH) { - tsdbError("vgId:%d invalid file %s exists, ignore it", REPO_ID(pRepo), dp->d_name); - continue; - } else { - goto _err; - } - } else { - goto _err; - } + ASSERT(TSDB_FILE_OPENED(pMFile)); - pFileH->pFGroup[pFileH->nFGroups++] = fileGroup; - qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); - tsdbDebug("vgId:%d file group %d is restored, nFGroups %d", REPO_ID(pRepo), fileGroup.fileId, pFileH->nFGroups); + if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) { + return -1; } - regfree(®ex1); - regfree(®ex2); - tfree(tDataDir); - closedir(dir); - return 0; - -_err: - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { + return -1; + } - regfree(®ex1); - regfree(®ex2); + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } - tfree(tDataDir); - if (dir != NULL) closedir(dir); - tsdbCloseFileH(pRepo); - return -1; + tsdbDecodeMFInfo(buf, pInfo); + return 0; } -void tsdbCloseFileH(STsdbRepo *pRepo) { - STsdbFileH *pFileH = pRepo->tsdbFileH; +int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) { + SMFile * pMFile = pRepo->fs->cstatus->pmf; + struct stat mfstat; + SMFile mf; - for (int i = 0; i < pFileH->nFGroups; i++) { - SFileGroup *pFGroup = pFileH->pFGroup + i; - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - tsdbDestroyFile(&pFGroup->files[type]); - } + if (pMFile == NULL) { + // No meta file, no need to scan + return 0; } -} -SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { - STsdbFileH *pFileH = pRepo->tsdbFileH; - STsdbCfg * pCfg = &(pRepo->config); + tsdbInitMFileEx(&mf, pMFile); - if (pFileH->nFGroups >= pFileH->maxFGroups) { - int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); - if (pFileH->pFGroup[0].fileId < mfid) { - pthread_rwlock_wrlock(&pFileH->fhlock); - tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[0])); - pthread_rwlock_unlock(&pFileH->fhlock); - } + if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) { + tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pMFile)); + pRepo->state |= TSDB_STATE_BAD_META; + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD); + return 0; } - ASSERT(pFileH->nFGroups < pFileH->maxFGroups); + if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } - SFileGroup fGroup; - SFileGroup *pFGroup = &fGroup; + if (pMFile->info.size < mfstat.st_size) { + if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { + return -1; + } - SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); - if (pGroup == NULL) { // if not exists, create one - pFGroup->fileId = fid; - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) { - for (int i = type; i >= 0; i--) { - remove(pFGroup->files[i].fname); - } + if (taosFtruncate(mf.fd, mf.info.size) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseMFile(&mf); + return -1; + } - return NULL; - } + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbCloseMFile(&mf); + return -1; } - pthread_rwlock_wrlock(&pFileH->fhlock); - pFileH->pFGroup[pFileH->nFGroups++] = fGroup; - qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); - pthread_rwlock_unlock(&pFileH->fhlock); - pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); - ASSERT(pGroup != NULL); + tsdbCloseMFile(&mf); + tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + mfstat.st_size, pMFile->info.size); + } else if (pMFile->info.size > mfstat.st_size) { + tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size); + pRepo->state |= TSDB_STATE_BAD_META; + TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_BAD); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return 0; + } else { + tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile)); } - return pGroup; + return 0; } -void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { - pIter->pFileH = pFileH; - pIter->direction = direction; +int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) { + int tlen = 0; - if (pFileH->nFGroups == 0) { - pIter->index = -1; - pIter->fileId = -1; - } else { - if (direction == TSDB_FGROUP_ITER_FORWARD) { - pIter->index = 0; - } else { - pIter->index = pFileH->nFGroups - 1; - } - pIter->fileId = pFileH->pFGroup[pIter->index].fileId; - } + tlen += taosEncodeVariantI64(buf, pInfo->size); + tlen += taosEncodeVariantI64(buf, pInfo->tombSize); + tlen += taosEncodeVariantI64(buf, pInfo->nRecords); + tlen += taosEncodeVariantI64(buf, pInfo->nDels); + tlen += taosEncodeFixedU32(buf, pInfo->magic); + + return tlen; +} + +void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) { + buf = taosDecodeVariantI64(buf, &(pInfo->size)); + buf = taosDecodeVariantI64(buf, &(pInfo->tombSize)); + buf = taosDecodeVariantI64(buf, &(pInfo->nRecords)); + buf = taosDecodeVariantI64(buf, &(pInfo->nDels)); + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); + + return buf; } -void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { - STsdbFileH *pFileH = pIter->pFileH; +static int tsdbRollBackMFile(SMFile *pMFile) { + SMFile mf; - if (pFileH->nFGroups == 0) { - pIter->index = -1; - pIter->fileId = -1; - return; + tsdbInitMFileEx(&mf, pMFile); + + if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { + return -1; } - int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; - void *ptr = taosbsearch(&fid, (void *)pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); - if (ptr == NULL) { - pIter->index = -1; - pIter->fileId = -1; - } else { - pIter->index = (int)(POINTER_DISTANCE(ptr, pFileH->pFGroup) / sizeof(SFileGroup)); - pIter->fileId = ((SFileGroup *)ptr)->fileId; + if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseMFile(&mf); + return -1; } + + if (tsdbUpdateMFileHeader(&mf) < 0) { + tsdbCloseMFile(&mf); + return -1; + } + + TSDB_FILE_FSYNC(&mf); + + tsdbCloseMFile(&mf); + return 0; } -SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { - STsdbFileH *pFileH = pIter->pFileH; - SFileGroup *pFGroup = NULL; +// ============== Operations on SDFile +void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) { + char fname[TSDB_FILENAME_LEN]; - if (pIter->index < 0 || pIter->index >= pFileH->nFGroups || pIter->fileId < 0) return NULL; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_OK); - pFGroup = &pFileH->pFGroup[pIter->index]; - if (pFGroup->fileId != pIter->fileId) { - tsdbSeekFileGroupIter(pIter, pIter->fileId); - } + TSDB_FILE_SET_CLOSED(pDFile); - if (pIter->index < 0) return NULL; + memset(&(pDFile->info), 0, sizeof(pDFile->info)); + pDFile->info.magic = TSDB_FILE_INIT_MAGIC; - pFGroup = &pFileH->pFGroup[pIter->index]; - ASSERT(pFGroup->fileId == pIter->fileId); + tsdbGetFilename(vid, fid, ver, ftype, fname); + tfsInitFile(&(pDFile->f), did.level, did.id, fname); +} - if (pIter->direction == TSDB_FGROUP_ITER_FORWARD) { - pIter->index++; - } else { - pIter->index--; - } +void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) { + *pDFile = *pODFile; + TSDB_FILE_SET_CLOSED(pDFile); +} - if (pIter->index >= 0 && pIter->index < pFileH->nFGroups) { - pIter->fileId = pFileH->pFGroup[pIter->index].fileId; - } else { - pIter->fileId = -1; - } +int tsdbEncodeSDFile(void **buf, SDFile *pDFile) { + int tlen = 0; - return pFGroup; + tlen += tsdbEncodeDFInfo(buf, &(pDFile->info)); + tlen += tfsEncodeFile(buf, &(pDFile->f)); + + return tlen; } -int tsdbOpenFile(SFile *pFile, int oflag) { - ASSERT(!TSDB_IS_FILE_OPENED(pFile)); +void *tsdbDecodeSDFile(void *buf, SDFile *pDFile) { + buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); + buf = tfsDecodeFile(buf, &(pDFile->f)); + TSDB_FILE_SET_CLOSED(pDFile); - pFile->fd = open(pFile->fname, oflag, 0755); - if (pFile->fd < 0) { - tsdbError("failed to open file %s since %s", pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + return buf; +} - tsdbTrace("open file %s, fd %d", pFile->fname, pFile->fd); +static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) { + int tlen = 0; - return 0; + tlen += tsdbEncodeDFInfo(buf, &(pDFile->info)); + tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pDFile)); + + return tlen; } -void tsdbCloseFile(SFile *pFile) { - if (TSDB_IS_FILE_OPENED(pFile)) { - tsdbTrace("close file %s, fd %d", pFile->fname, pFile->fd); - close(pFile->fd); - pFile->fd = -1; - } +static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { + char *aname; + + buf = tsdbDecodeDFInfo(buf, &(pDFile->info)); + buf = taosDecodeString(buf, &aname); + strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN); + TSDB_FILE_SET_CLOSED(pDFile); + tfree(aname); + + return buf; } -int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { - memset((void *)pFile, 0, sizeof(SFile)); - pFile->fd = -1; +int tsdbCreateDFile(SDFile *pDFile, bool updateHeader) { + ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); - tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), fid, type, pFile->fname); + pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (pDFile->fd < 0) { + if (errno == ENOENT) { + // Try to create directory recursively + char *s = strdup(TFILE_REL_NAME(&(pDFile->f))); + if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pDFile), TSDB_FILE_ID(pDFile)) < 0) { + tfree(s); + return -1; + } + tfree(s); - if (access(pFile->fname, F_OK) == 0) { - tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), pFile->fname); - terrno = TSDB_CODE_TDB_FILE_ALREADY_EXISTS; - goto _err; + pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); + if (pDFile->fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } } - if (tsdbOpenFile(pFile, O_RDWR | O_CREAT) < 0) { - goto _err; + if (!updateHeader) { + return 0; } - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; + pDFile->info.size += TSDB_FILE_HEAD_SIZE; - if (tsdbUpdateFileHeader(pFile) < 0) { - tsdbCloseFile(pFile); + if (tsdbUpdateDFileHeader(pDFile) < 0) { + tsdbCloseDFile(pDFile); + tsdbRemoveDFile(pDFile); return -1; } - tsdbCloseFile(pFile); - return 0; - -_err: - return -1; } -SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid, int flags) { - void *ptr = - taosbsearch((void *)(&fid), (void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); - if (ptr == NULL) return NULL; - return (SFileGroup *)ptr; +int tsdbUpdateDFileHeader(SDFile *pDFile) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; + + if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) { + return -1; + } + + void *ptr = buf; + taosEncodeFixedU32(&ptr, TSDB_FS_VERSION); + tsdbEncodeDFInfo(&ptr, &(pDFile->info)); + + taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + if (tsdbWriteDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { + return -1; + } + + return 0; } -void tsdbFitRetention(STsdbRepo *pRepo) { - STsdbCfg *pCfg = &(pRepo->config); - STsdbFileH *pFileH = pRepo->tsdbFileH; - SFileGroup *pGroup = pFileH->pFGroup; +int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; + uint32_t version; - int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile); + ASSERT(TSDB_FILE_OPENED(pDFile)); - pthread_rwlock_wrlock(&(pFileH->fhlock)); + if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) { + return -1; + } - while (pFileH->nFGroups > 0 && pGroup[0].fileId < mfid) { - tsdbRemoveFileGroup(pRepo, pGroup); + if (tsdbReadDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) { + return -1; } - pthread_rwlock_unlock(&(pFileH->fhlock)); + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + void *pBuf = buf; + pBuf = taosDecodeFixedU32(pBuf, &version); + pBuf = tsdbDecodeDFInfo(pBuf, pInfo); + return 0; } -int tsdbUpdateFileHeader(SFile *pFile) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; +static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) { + struct stat dfstat; + SDFile df; - void *pBuf = (void *)buf; - taosEncodeFixedU32((void *)(&pBuf), TSDB_FILE_VERSION); - tsdbEncodeSFileInfo((void *)(&pBuf), &(pFile->info)); + tsdbInitDFileEx(&df, pDFile); - taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) { + tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pDFile)); + pRepo->state |= TSDB_STATE_BAD_DATA; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); + return 0; + } - if (lseek(pFile->fd, 0, SEEK_SET) < 0) { - tsdbError("failed to lseek file %s since %s", pFile->fname, strerror(errno)); + if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (taosWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + + if (pDFile->info.size < dfstat.st_size) { + if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + return -1; + } + + if (taosFtruncate(df.fd, df.info.size) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseDFile(&df); + return -1; + } + + if (tsdbUpdateDFileHeader(&df) < 0) { + tsdbCloseDFile(&df); + return -1; + } + + tsdbCloseDFile(&df); + tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + dfstat.st_size, pDFile->info.size); + } else if (pDFile->info.size > dfstat.st_size) { + tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size); + pRepo->state |= TSDB_STATE_BAD_DATA; + TSDB_FILE_SET_STATE(pDFile, TSDB_FILE_STATE_BAD); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return 0; + } else { + tsdbDebug("vgId:%d file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); } return 0; } -int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { +static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) { int tlen = 0; + tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); @@ -450,7 +499,7 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { return tlen; } -void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { +static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); @@ -462,156 +511,187 @@ void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { return buf; } -void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { - ASSERT(pFGroup != NULL); - STsdbFileH *pFileH = pRepo->tsdbFileH; - - SFileGroup fileGroup = *pFGroup; +static int tsdbApplyDFileChange(SDFile *from, SDFile *to) { + ASSERT(from != NULL || to != NULL); - int nFilesLeft = pFileH->nFGroups - (int)(POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1); - if (nFilesLeft > 0) { - memmove((void *)pFGroup, POINTER_SHIFT(pFGroup, sizeof(SFileGroup)), sizeof(SFileGroup) * nFilesLeft); - } - - pFileH->nFGroups--; - ASSERT(pFileH->nFGroups >= 0); - - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - if (remove(fileGroup.files[type].fname) < 0) { - tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); + if (from != NULL) { + if (to == NULL) { + tsdbRemoveDFile(from); + } else { + if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) { + if (from->info.size > to->info.size) { + tsdbRollBackDFile(to); + } + } else { + (void)tsdbRemoveDFile(from); + } } - tsdbDestroyFile(&fileGroup.files[type]); } + + return 0; } -int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; +static int tsdbRollBackDFile(SDFile *pDFile) { + SDFile df = *pDFile; - if (lseek(pFile->fd, 0, SEEK_SET) < 0) { - tsdbError("failed to lseek file %s to start since %s", pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + if (tsdbOpenDFile(&df, O_WRONLY) < 0) { return -1; } - if (taosRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("failed to read file %s header part with %d bytes, reason:%s", pFile->fname, TSDB_FILE_HEAD_SIZE, - strerror(errno)); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + if (taosFtruncate(TSDB_FILE_FD(&df), pDFile->info.size) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseDFile(&df); return -1; } - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { - tsdbError("file %s header part is corrupted with failed checksum", pFile->fname); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + if (tsdbUpdateDFileHeader(&df) < 0) { + tsdbCloseDFile(&df); return -1; } - void *pBuf = (void *)buf; - pBuf = taosDecodeFixedU32(pBuf, version); - pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); + TSDB_FILE_FSYNC(&df); + tsdbCloseDFile(&df); return 0; } -void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) { - uint32_t version = 0; - SFile file; - SFile * pFile = &file; +// ============== Operations on SDFileSet +void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t ver) { + pSet->fid = fid; + pSet->state = 0; - strncpy(pFile->fname, fname, TSDB_FILENAME_LEN - 1); - pFile->fd = -1; - - if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err; - if (tsdbLoadFileHeader(pFile, &version) < 0) goto _err; + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype); + tsdbInitDFile(pDFile, did, vid, fid, ver, ftype); + } +} - off_t offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) goto _err; - tsdbCloseFile(pFile); +void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) { + pSet->fid = pOSet->fid; + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype)); + } +} - *magic = pFile->info.magic; - *size = offset; +int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) { + int tlen = 0; - return; + tlen += taosEncodeFixedI32(buf, pSet->fid); + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype)); + } -_err: - tsdbCloseFile(pFile); - *magic = TSDB_FILE_INIT_MAGIC; - *size = 0; + return tlen; } -// ---------------- LOCAL FUNCTIONS ---------------- -static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { - uint32_t version; +void *tsdbDecodeDFileSet(void *buf, SDFileSet *pSet) { + int32_t fid; - tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), fid, type, pFile->fname); + buf = taosDecodeFixedI32(buf, &(fid)); + pSet->state = 0; + pSet->fid = fid; + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + buf = tsdbDecodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype)); + } + return buf; +} - pFile->fd = -1; - if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err; +int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) { + int tlen = 0; - if (tsdbLoadFileHeader(pFile, &version) < 0) { - tsdbError("vgId:%d failed to load file %s header part since %s", REPO_ID(pRepo), pFile->fname, tstrerror(terrno)); - goto _err; + tlen += taosEncodeFixedI32(buf, pSet->fid); + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } - if (pFile->info.size == TSDB_FILE_HEAD_SIZE) { - pFile->info.size = lseek(pFile->fd, 0, SEEK_END); - } + return tlen; +} + +void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) { + int32_t fid; - if (version != TSDB_FILE_VERSION) { - // TODO: deal with error - tsdbError("vgId:%d file %s version %u is not the same as program version %u which may cause problem", - REPO_ID(pRepo), pFile->fname, version, TSDB_FILE_VERSION); + buf = taosDecodeFixedI32(buf, &(fid)); + pSet->fid = fid; + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype)); } + return buf; +} - tsdbCloseFile(pFile); +int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFileFrom = (from) ? TSDB_DFILE_IN_SET(from, ftype) : NULL; + SDFile *pDFileTo = (to) ? TSDB_DFILE_IN_SET(to, ftype) : NULL; + if (tsdbApplyDFileChange(pDFileFrom, pDFileTo) < 0) { + return -1; + } + } return 0; -_err: - tsdbDestroyFile(pFile); - return -1; } -static void tsdbDestroyFile(SFile *pFile) { tsdbCloseFile(pFile); } +int tsdbCreateDFileSet(SDFileSet *pSet, bool updateHeader) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbCreateDFile(TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) { + tsdbCloseDFileSet(pSet); + tsdbRemoveDFileSet(pSet); + return -1; + } + } -static int compFGroup(const void *arg1, const void *arg2) { - int val1 = ((SFileGroup *)arg1)->fileId; - int val2 = ((SFileGroup *)arg2)->fileId; + return 0; +} - if (val1 < val2) { - return -1; - } else if (val1 > val2) { - return 1; - } else { - return 0; +int tsdbUpdateDFileSetHeader(SDFileSet *pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbUpdateDFileHeader(TSDB_DFILE_IN_SET(pSet, ftype)) < 0) { + return -1; + } } + return 0; } -static int keyFGroupCompFunc(const void *key, const void *fgroup) { - int fid = *(int *)key; - SFileGroup *pFGroup = (SFileGroup *)fgroup; - if (fid == pFGroup->fileId) { - return 0; - } else { - return fid > pFGroup->fileId ? 1 : -1; +int tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet *pSet) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + if (tsdbScanAndTryFixDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype)) < 0) { + return -1; + } } + return 0; } -static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo) { - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbInitFile(&pFGroup->files[type], pRepo, pFGroup->fileId, type) < 0) { - memset(&pFGroup->files[type].info, 0, sizeof(STsdbFileInfo)); - pFGroup->files[type].info.magic = TSDB_FILE_INIT_MAGIC; - pFGroup->state = 1; - pRepo->state = TSDB_STATE_BAD_FILE; - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; +int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version) { + char *p = NULL; + *version = 0; + *ftype = TSDB_FILE_MAX; + + sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, version); + for (TSDB_FILE_T i = 0; i < TSDB_FILE_MAX; i++) { + if (strcmp(p, TSDB_FNAME_SUFFIX[i]) == 0) { + *ftype = i; + break; } } -} -static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep) { - return (TSKEY)(taosGetTimestamp(precision) - keep * tsMsPerDay[precision]); + tfree(p); + return 0; } -static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days) { - return (int)(TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision)); +static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) { + ASSERT(ftype != TSDB_FILE_MAX); + + if (ftype < TSDB_FILE_MAX) { + if (ver == 0) { + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]); + } else { + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid, + TSDB_FNAME_SUFFIX[ftype], ver); + } + } else { + if (ver == 0) { + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vid, TSDB_FNAME_SUFFIX[ftype]); + } else { + snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver); + } + } } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b34b2fa9e6d867527d98036da7972b50c5471c4b..8969f61596b33f37cabe934fdd56819b71575315 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -14,130 +14,108 @@ */ // no test file errors here -#include "tsdbMain.h" -#include "os.h" -#include "talgo.h" -#include "taosdef.h" -#include "tchecksum.h" -#include "tscompression.h" -#include "tsdb.h" -#include "tulog.h" - -#define TSDB_CFG_FILE_NAME "config" -#define TSDB_DATA_DIR_NAME "data" -#define TSDB_META_FILE_NAME "meta" -#define TSDB_META_FILE_INDEX 10000000 +#include "tsdbint.h" + #define IS_VALID_PRECISION(precision) \ (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) #define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); -static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg); -static int32_t tsdbUnsetRepoEnv(char *rootDir); -static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg); -static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg); -static char * tsdbGetCfgFname(char *rootDir); -static STsdbRepo * tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg); -static void tsdbFreeRepo(STsdbRepo *pRepo); -static int tsdbRestoreInfo(STsdbRepo *pRepo); -static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); -static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); -static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); -static int keyFGroupCompFunc(const void *key, const void *fgroup); -static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg); -static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg); -static void tsdbStartStream(STsdbRepo *pRepo); -static void tsdbStopStream(STsdbRepo *pRepo); +static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); +static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); +static void tsdbFreeRepo(STsdbRepo *pRepo); +static void tsdbStartStream(STsdbRepo *pRepo); +static void tsdbStopStream(STsdbRepo *pRepo); // Function declaration -int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { - DIR *dir = opendir(rootDir); - if (dir) { - tsdbDebug("repository %s already exists", rootDir); - closedir(dir); - return 0; - } else { - if (ENOENT != errno) { - tsdbError("failed to open directory %s since %s", rootDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } +int32_t tsdbCreateRepo(int repoid) { + char tsdbDir[TSDB_FILENAME_LEN] = "\0"; + char dataDir[TSDB_FILENAME_LEN] = "\0"; - if (mkdir(rootDir, 0755) < 0) { - tsdbError("vgId:%d failed to create rootDir %s since %s", pCfg->tsdbId, rootDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + tsdbGetRootDir(repoid, tsdbDir); + if (tfsMkdir(tsdbDir) < 0) { + goto _err; } - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; + tsdbGetDataDir(repoid, dataDir); + if (tfsMkdir(dataDir) < 0) { + goto _err; + } - if (tsdbSetRepoEnv(rootDir, pCfg) < 0) return -1; + // TODO: need to create current file with nothing in - tsdbDebug( - "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d daysPerFile %d keep " - "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d update %d cacheLastRow %d", - pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, - pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression, pCfg->update, pCfg->cacheLastRow); return 0; + +_err: + tsdbError("vgId:%d failed to create TSDB repository since %s", repoid, tstrerror(terrno)); + return -1; } -int32_t tsdbDropRepo(char *rootDir) { return tsdbUnsetRepoEnv(rootDir); } +int32_t tsdbDropRepo(int repoid) { + char tsdbDir[TSDB_FILENAME_LEN] = "\0"; -TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { - STsdbCfg config = {0}; - STsdbRepo *pRepo = NULL; + tsdbGetRootDir(repoid, tsdbDir); + return tfsRmdir(tsdbDir); +} + +STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { + STsdbRepo *pRepo; + STsdbCfg config = *pCfg; terrno = TSDB_CODE_SUCCESS; - if (tsdbLoadConfig(rootDir, &config) < 0) { - tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno)); + // Check and set default configurations + if (tsdbCheckAndSetDefaultCfg(&config) < 0) { + tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno)); return NULL; } - pRepo = tsdbNewRepo(rootDir, pAppH, &config); - if (pRepo == NULL) { - tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno)); + // Create new TSDB object + if ((pRepo = tsdbNewRepo(&config, pAppH)) == NULL) { + tsdbError("vgId:%d failed to open TSDB repository while creating TSDB object since %s", config.tsdbId, + tstrerror(terrno)); return NULL; } + // Open meta if (tsdbOpenMeta(pRepo) < 0) { - tsdbError("vgId:%d failed to open meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + tsdbError("vgId:%d failed to open TSDB repository while opening Meta since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; } if (tsdbOpenBufPool(pRepo) < 0) { - tsdbError("vgId:%d failed to open buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + tsdbError("vgId:%d failed to open TSDB repository while opening buffer pool since %s", config.tsdbId, + tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; } - if (tsdbOpenFileH(pRepo) < 0) { - tsdbError("vgId:%d failed to open file handle since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + if (tsdbOpenFS(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while opening FS since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; } - if (tsdbRestoreInfo(pRepo) < 0) { - tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + // TODO: Restore information from data + if ((!(pRepo->state & TSDB_STATE_BAD_DATA)) && tsdbRestoreInfo(pRepo) < 0) { + tsdbError("vgId:%d failed to open TSDB repository while restore info since %s", config.tsdbId, tstrerror(terrno)); + tsdbCloseRepo(pRepo, false); + return NULL; } tsdbStartStream(pRepo); - tsdbDebug("vgId:%d open tsdb repository succeed!", REPO_ID(pRepo)); - - return (TSDB_REPO_T *)pRepo; + tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo)); -_err: - tsdbCloseRepo(pRepo, false); - return NULL; + return pRepo; } // Note: all working thread and query thread must stopped when calling this function -int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { +int tsdbCloseRepo(STsdbRepo *repo, int toCommit) { if (repo == NULL) return 0; - STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbRepo *pRepo = repo; int vgId = REPO_ID(pRepo); terrno = TSDB_CODE_SUCCESS; @@ -145,16 +123,15 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { tsdbStopStream(pRepo); if (toCommit) { - tsdbAsyncCommit(pRepo); - sem_wait(&(pRepo->readyToCommit)); - terrno = pRepo->code; + tsdbSyncCommit(repo); } + tsdbUnRefMemTable(pRepo, pRepo->mem); tsdbUnRefMemTable(pRepo, pRepo->imem); pRepo->mem = NULL; pRepo->imem = NULL; - tsdbCloseFileH(pRepo); + tsdbCloseFS(pRepo); tsdbCloseBufPool(pRepo); tsdbCloseMeta(pRepo); tsdbFreeRepo(pRepo); @@ -167,88 +144,67 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { } } -uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - // STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - uint32_t magic = 0; - char * fname = NULL; - - struct stat fState; +STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) { + ASSERT(repo != NULL); + return &((STsdbRepo *)repo)->config; +} - tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); - ASSERT(*index <= eindex); +int tsdbLockRepo(STsdbRepo *pRepo) { + int code = pthread_mutex_lock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + pRepo->repoLocked = true; + return 0; +} - char *sdup = strdup(pRepo->rootDir); - char *prefix = dirname(sdup); - int prefixLen = (int)strlen(prefix); +int tsdbUnlockRepo(STsdbRepo *pRepo) { + ASSERT(IS_REPO_LOCKED(pRepo)); + pRepo->repoLocked = false; + int code = pthread_mutex_unlock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} - if (name[0] == 0) { // get the file from index or after, but not larger than eindex - tfree(sdup); - int fid = (*index) / TSDB_FILE_TYPE_MAX; +int tsdbCheckCommit(STsdbRepo *pRepo) { + ASSERT(pRepo->mem != NULL); + STsdbCfg *pCfg = &(pRepo->config); - if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { - if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { - fname = tsdbGetMetaFileName(pRepo->rootDir); - *index = TSDB_META_FILE_INDEX; - magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); - } else { - return 0; - } - } else { - SFileGroup *pFGroup = - taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); - if (pFGroup->fileId == fid) { - fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname); - magic = pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].info.magic; - } else { - if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { - fname = strdup(pFGroup->files[0].fname); - *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; - magic = pFGroup->files[0].info.magic; - } else { - return 0; - } - } - } - strcpy(name, fname + prefixLen); - } else { // get the named file at the specified index. If not there, return 0 - fname = malloc(prefixLen + strlen(name) + 2); - sprintf(fname, "%s/%s", prefix, name); - if (access(fname, F_OK) != 0) { - tfree(fname); - tfree(sdup); - return 0; - } - if (*index == TSDB_META_FILE_INDEX) { // get meta file - tsdbGetStoreInfo(fname, &magic, size); - } else { - tsdbGetFileInfoImpl(fname, &magic, size); - } - tfree(fname); - tfree(sdup); - return magic; + STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); + ASSERT(pBufBlock != NULL); + if ((pRepo->mem->extraBuffList != NULL) || + ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { + // trigger commit + if (tsdbAsyncCommit(pRepo) < 0) return -1; } - if (stat(fname, &fState) < 0) { - tfree(fname); - return 0; - } + return 0; +} - *size = fState.st_size; - // magic = *size; +STsdbMeta *tsdbGetMeta(STsdbRepo *pRepo) { return pRepo->tsdbMeta; } - tfree(fname); - return magic; -} +STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; } -STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { +int tsdbGetState(STsdbRepo *repo) { return repo->state; } + +void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { ASSERT(repo != NULL); - return &((STsdbRepo *)repo)->config; + STsdbRepo *pRepo = repo; + *totalPoints = pRepo->stat.pointsWritten; + *totalStorage = pRepo->stat.totalStorage; + *compStorage = pRepo->stat.compStorage; } -int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { +int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { // TODO: think about multithread cases + return 0; +#if 0 STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbCfg config = pRepo->config; STsdbCfg * pRCfg = &pRepo->config; @@ -294,112 +250,115 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { } return 0; +#endif } -void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { - ASSERT(repo != NULL); - STsdbRepo *pRepo = repo; - *totalPoints = pRepo->stat.pointsWritten; - *totalStorage = pRepo->stat.totalStorage; - *compStorage = pRepo->stat.compStorage; -} - -int tsdbGetState(TSDB_REPO_T *repo) { - return ((STsdbRepo *)repo)->state; -} +uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) { + // TODO + return 0; +#if 0 + STsdbRepo *pRepo = (STsdbRepo *)repo; + // STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + uint32_t magic = 0; + char * fname = NULL; -// ----------------- INTERNAL FUNCTIONS ----------------- -char *tsdbGetMetaFileName(char *rootDir) { - int tlen = (int)(strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2); - char *fname = calloc(1, tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } + struct stat fState; - snprintf(fname, tlen, "%s/%s", rootDir, TSDB_META_FILE_NAME); - return fname; -} + tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex); + ASSERT(*index <= eindex); -void tsdbGetDataFileName(char *rootDir, int vid, int fid, int type, char *fname) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d%s", rootDir, TSDB_DATA_DIR_NAME, vid, fid, tsdbFileSuffix[type]); -} + if (name[0] == 0) { // get the file from index or after, but not larger than eindex + int fid = (*index) / TSDB_FILE_TYPE_MAX; -int tsdbLockRepo(STsdbRepo *pRepo) { - int code = pthread_mutex_lock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; + if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { + if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { + fname = tsdbGetMetaFileName(pRepo->rootDir); + *index = TSDB_META_FILE_INDEX; + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); + sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME); + } else { + return 0; + } + } else { + SFileGroup *pFGroup = + taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); + if (pFGroup->fileId == fid) { + SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; + fname = strdup(TSDB_FILE_NAME(pFile)); + magic = pFile->info.magic; + char *tfname = strdup(fname); + sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); + tfree(tfname); + } else { + if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { + SFile *pFile = &pFGroup->files[0]; + fname = strdup(TSDB_FILE_NAME(pFile)); + *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; + magic = pFile->info.magic; + char *tfname = strdup(fname); + sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname)); + tfree(tfname); + } else { + return 0; + } + } + } + } else { // get the named file at the specified index. If not there, return 0 + fname = malloc(256); + sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name); + if (access(fname, F_OK) != 0) { + tfree(fname); + return 0; + } + if (*index == TSDB_META_FILE_INDEX) { // get meta file + tsdbGetStoreInfo(fname, &magic, size); + } else { + char tfname[TSDB_FILENAME_LEN] = "\0"; + sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name)); + tsdbGetFileInfoImpl(tfname, &magic, size); + } + tfree(fname); + return magic; } - pRepo->repoLocked = true; - return 0; -} -int tsdbUnlockRepo(STsdbRepo *pRepo) { - ASSERT(IS_REPO_LOCKED(pRepo)); - pRepo->repoLocked = false; - int code = pthread_mutex_unlock(&pRepo->mutex); - if (code != 0) { - tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - return -1; + if (stat(fname, &fState) < 0) { + tfree(fname); + return 0; } - return 0; -} -char *tsdbGetDataDirName(char *rootDir) { - int tlen = (int)(strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2); - char *fname = calloc(1, tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } + *size = fState.st_size; + // magic = *size; - snprintf(fname, tlen, "%s/%s", rootDir, TSDB_DATA_DIR_NAME); - return fname; + tfree(fname); + return magic; +#endif } -int tsdbGetNextMaxTables(int tid) { - ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); - int maxTables = TSDB_INIT_NTABLES; - while (true) { - maxTables = MIN(maxTables, TSDB_MAX_TABLES); - if (tid <= maxTables) break; - maxTables *= 2; - } - - return maxTables + 1; +void tsdbGetRootDir(int repoid, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid); } -int tsdbCheckCommit(STsdbRepo *pRepo) { - ASSERT(pRepo->mem != NULL); - STsdbCfg *pCfg = &(pRepo->config); - - STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); - ASSERT(pBufBlock != NULL); - if ((pRepo->mem->extraBuffList != NULL) || - ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) { - // trigger commit - if (tsdbAsyncCommit(pRepo) < 0) return -1; - } - - return 0; +void tsdbGetDataDir(int repoid, char dirName[]) { + snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid); } -STsdbMeta * tsdbGetMeta(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbMeta; } -STsdbFileH * tsdbGetFile(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbFileH; } -STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; } - -// ----------------- LOCAL FUNCTIONS ----------------- static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { + // Check tsdbId + if (pCfg->tsdbId < 0) { + tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; + } + // Check precision if (pCfg->precision == -1) { pCfg->precision = TSDB_DEFAULT_PRECISION; } else { if (!IS_VALID_PRECISION(pCfg->precision)) { tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } @@ -409,16 +368,11 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } else { if (!IS_VALID_COMPRESSION(pCfg->compression)) { tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } - // Check tsdbId - if (pCfg->tsdbId < 0) { - tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); - goto _err; - } - // Check daysPerFile if (pCfg->daysPerFile == -1) { pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; @@ -428,7 +382,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " "%d", pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } @@ -441,7 +396,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " "TSDB_MAX_MIN_ROW_FBLOCK %d", pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } @@ -453,14 +409,16 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " "TSDB_MAX_MAX_ROW_FBLOCK %d", pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId, pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } // Check keep @@ -472,456 +430,193 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d " "TSDB_MAX_KEEP %d", pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); - goto _err; + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; } } - // update check - if (pCfg->update != 0) pCfg->update = 1; - - // update cacheLastRow - if (pCfg->cacheLastRow != 0) pCfg->cacheLastRow = 1; - - return 0; - -_err: - terrno = TSDB_CODE_TDB_INVALID_CONFIG; - return -1; -} - -static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg) { - if (tsdbSaveConfig(rootDir, pCfg) < 0) { - tsdbError("vgId:%d failed to set TSDB environment since %s", pCfg->tsdbId, tstrerror(terrno)); - return -1; - } - - char *dirName = tsdbGetDataDirName(rootDir); - if (dirName == NULL) return -1; - - if (mkdir(dirName, 0755) < 0) { - tsdbError("vgId:%d failed to create directory %s since %s", pCfg->tsdbId, dirName, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - free(dirName); - return -1; + if (pCfg->keep1 == 0) { + pCfg->keep1 = pCfg->keep; } - free(dirName); - - char *fname = tsdbGetMetaFileName(rootDir); - if (fname == NULL) return -1; - if (tdCreateKVStore(fname) < 0) { - tsdbError("vgId:%d failed to open KV store since %s", pCfg->tsdbId, tstrerror(terrno)); - free(fname); - return -1; - } - - free(fname); - return 0; -} - -static int32_t tsdbUnsetRepoEnv(char *rootDir) { - taosRemoveDir(rootDir); - tsdbDebug("repository %s is removed", rootDir); - return 0; -} - -static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { - int fd = -1; - char *fname = NULL; - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - char *pBuf = buf; - - fname = tsdbGetCfgFname(rootDir); - if (fname == NULL) { - tsdbError("vgId:%d failed to save configuration since %s", pCfg->tsdbId, tstrerror(terrno)); - goto _err; - } - - fd = open(fname, O_WRONLY | O_CREAT, 0755); - if (fd < 0) { - tsdbError("vgId:%d failed to open file %s since %s", pCfg->tsdbId, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - int tlen = tsdbEncodeCfg((void *)(&pBuf), pCfg); - ASSERT((tlen + sizeof(TSCKSUM) <= TSDB_FILE_HEAD_SIZE) && (POINTER_DISTANCE(pBuf, buf) == tlen)); - - taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); - - if (taosWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (pCfg->keep2 == 0) { + pCfg->keep2 = pCfg->keep; } - if (fsync(fd) < 0) { - tsdbError("vgId:%d failed to fsync file %s since %s", pCfg->tsdbId, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - free(fname); - close(fd); - return 0; - -_err: - tfree(fname); - if (fd >= 0) close(fd); - return -1; -} - -static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { - char *fname = NULL; - int fd = -1; - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - - fname = tsdbGetCfgFname(rootDir); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - fd = open(fname, O_RDONLY); - if (fd < 0) { - tsdbError("failed to open file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { - tsdbError("file %s is corrupted", fname); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - goto _err; - } - - tsdbDecodeCfg(buf, pCfg); + // update check + if (pCfg->update != 0) pCfg->update = 1; - tfree(fname); - close(fd); + // update cacheLastRow + if (pCfg->cacheLastRow != 0) pCfg->cacheLastRow = 1; return 0; - -_err: - tfree(fname); - if (fd >= 0) close(fd); - return -1; -} - -static char *tsdbGetCfgFname(char *rootDir) { - int tlen = (int)(strlen(rootDir) + strlen(TSDB_CFG_FILE_NAME) + 2); - char *fname = calloc(1, tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - snprintf(fname, tlen, "%s/%s", rootDir, TSDB_CFG_FILE_NAME); - return fname; } -static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo)); +static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { + STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo)); if (pRepo == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + return NULL; } pRepo->state = TSDB_STATE_OK; pRepo->code = TSDB_CODE_SUCCESS; + pRepo->config = *pCfg; + if (pAppH) { + pRepo->appH = *pAppH; + } + pRepo->repoLocked = false; - int code = pthread_mutex_init(&pRepo->mutex, NULL); + int code = pthread_mutex_init(&(pRepo->mutex), NULL); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); - goto _err; + tsdbFreeRepo(pRepo); + return NULL; } - code = sem_init(&(pRepo->readyToCommit), 0, 1); + code = tsem_init(&(pRepo->readyToCommit), 0, 1); if (code != 0) { + code = errno; terrno = TAOS_SYSTEM_ERROR(code); - goto _err; - } - - pRepo->repoLocked = false; - - pRepo->rootDir = strdup(rootDir); - if (pRepo->rootDir == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + tsdbFreeRepo(pRepo); + return NULL; } - pRepo->config = *pCfg; - if (pAppH) pRepo->appH = *pAppH; - pRepo->tsdbMeta = tsdbNewMeta(pCfg); if (pRepo->tsdbMeta == NULL) { tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + tsdbFreeRepo(pRepo); + return NULL; } pRepo->pPool = tsdbNewBufPool(pCfg); if (pRepo->pPool == NULL) { tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + tsdbFreeRepo(pRepo); + return NULL; } - pRepo->tsdbFileH = tsdbNewFileH(pCfg); - if (pRepo->tsdbFileH == NULL) { - tsdbError("vgId:%d failed to create file handle since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; + pRepo->fs = tsdbNewFS(pCfg); + if (pRepo->fs == NULL) { + tsdbError("vgId:%d failed to TSDB file system since %s", REPO_ID(pRepo), tstrerror(terrno)); + tsdbFreeRepo(pRepo); + return NULL; } return pRepo; - -_err: - tsdbFreeRepo(pRepo); - return NULL; } static void tsdbFreeRepo(STsdbRepo *pRepo) { if (pRepo) { - tsdbFreeFileH(pRepo->tsdbFileH); + tsdbFreeFS(pRepo->fs); tsdbFreeBufPool(pRepo->pPool); tsdbFreeMeta(pRepo->tsdbMeta); // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); - tfree(pRepo->rootDir); - sem_destroy(&(pRepo->readyToCommit)); + tsem_destroy(&(pRepo->readyToCommit)); pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } } -static int tsdbRestoreInfo(STsdbRepo *pRepo) { // TODO - STsdbMeta * pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - SFileGroup *pFGroup = NULL; - STsdbCfg * pCfg = &(pRepo->config); - SCompBlock *pBlock = NULL; +static void tsdbStartStream(STsdbRepo *pRepo) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + + for (int i = 0; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->type == TSDB_STREAM_TABLE) { + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, + tsdbGetTableSchemaImpl(pTable, false, false, -1)); + } + } +} + +static void tsdbStopStream(STsdbRepo *pRepo) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + + for (int i = 0; i < pMeta->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->type == TSDB_STREAM_TABLE) { + (*pRepo->appH.cqDropFunc)(pTable->cqhandle); + } + } +} + +int tsdbRestoreInfo(STsdbRepo *pRepo) { + SFSIter fsiter; + SReadH readh; + SDFileSet *pSet; + STsdbMeta *pMeta = pRepo->tsdbMeta; + STsdbCfg * pCfg = REPO_CFG(pRepo); + SBlock * pBlock; - SFileGroupIter iter; - SRWHelper rhelper = {0}; + if (tsdbInitReadH(&readh, pRepo) < 0) { + return -1; + } - if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; + tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD); + + while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) { + if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + if (tsdbLoadBlockIdx(&readh) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } - tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_DESC); - while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { - if (pFGroup->state) continue; - if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; - if (tsdbLoadCompIdx(&rhelper, NULL) < 0) goto _err; for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; - if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err; - SCompIdx *pIdx = &(rhelper.curCompIdx); - TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); - if (pIdx->offset > 0 && lastKey < pIdx->maxKey) { + if (tsdbSetReadTable(&readh, pTable) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable); + SBlockIdx *pIdx = readh.pBlkIdx; + if (pIdx && lastKey < pIdx->maxKey) { pTable->lastKey = pIdx->maxKey; - if (pCfg->cacheLastRow) { // load the block of data - if (tsdbLoadCompInfo(&rhelper, NULL) < 0) goto _err; - pBlock = rhelper.pCompInfo->blocks + pIdx->numOfBlocks - 1; - if (tsdbLoadBlockData(&rhelper, pBlock, NULL) < 0) goto _err; + if (pCfg->cacheLastRow) { + if (tsdbLoadBlockInfo(&readh, NULL) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } - // construct the data row + pBlock = readh.pBlkInfo->blocks + pIdx->numOfBlocks - 1; + + if (tsdbLoadBlockData(&readh, pBlock, NULL) < 0) { + tsdbDestroyReadH(&readh); + return -1; + } + + // Get the data in row ASSERT(pTable->lastRow == NULL); STSchema *pSchema = tsdbGetTableSchema(pTable); - pTable->lastRow = taosTMalloc(schemaTLen(pSchema)); + pTable->lastRow = taosTMalloc(dataRowMaxBytesFromSchema(pSchema)); if (pTable->lastRow == NULL) { - goto _err; + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyReadH(&readh); + return -1; } tdInitDataRow(pTable->lastRow, pSchema); for (int icol = 0; icol < schemaNCols(pSchema); icol++) { STColumn *pCol = schemaColAt(pSchema, icol); - SDataCol *pDataCol = rhelper.pDataCols[0]->cols + icol; + SDataCol *pDataCol = readh.pDCols[0]->cols + icol; tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes, pCol->offset); } } } + } } - tsdbDestroyHelper(&rhelper); + tsdbDestroyReadH(&readh); return 0; - -_err: - tsdbDestroyHelper(&rhelper); - return -1; -} - -static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { - int8_t ocompression = pRepo->config.compression; - pRepo->config.compression = compression; - tsdbDebug("vgId:%d tsdb compression is changed from %d to %d", REPO_ID(pRepo), ocompression, compression); -} - -static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { - STsdbCfg * pCfg = &pRepo->config; - STsdbFileH *pFileH = pRepo->tsdbFileH; - int okeep = pCfg->keep; - SFileGroup *pFGroup = NULL; - - ASSERT(pCfg->keep != keep); - int maxFiles = TSDB_MAX_FILE(keep, pCfg->daysPerFile); - - if (maxFiles != pFileH->maxFGroups) { - pthread_rwlock_wrlock(&(pFileH->fhlock)); - - pCfg->keep = keep; - pFGroup = (SFileGroup *)calloc(maxFiles, sizeof(SFileGroup)); - if (pFGroup == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - pthread_rwlock_unlock(&(pFileH->fhlock)); - return -1; - } - - int mfid = (int)(TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - - TSDB_MAX_FILE(keep, pCfg->daysPerFile)); - - int i = 0; - for (; i < pFileH->nFGroups; i++) { - if (pFileH->pFGroup[i].fileId >= mfid) break; - tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[i])); - } - - for (int j = 0; i < pFileH->nFGroups; i++, j++) { - pFGroup[j] = pFileH->pFGroup[i]; - } - - free(pFileH->pFGroup); - pFileH->pFGroup = pFGroup; - - pthread_rwlock_unlock(&(pFileH->fhlock)); - } - - tsdbDebug("vgId:%d keep is changed from %d to %d", REPO_ID(pRepo), okeep, keep); - - return 0; -} - -static int keyFGroupCompFunc(const void *key, const void *fgroup) { - int fid = *(int *)key; - SFileGroup *pFGroup = (SFileGroup *)fgroup; - if (fid == pFGroup->fileId) { - return 0; - } else { - return fid > pFGroup->fileId ? 1 : -1; - } -} - -static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { - int tlen = 0; - - tlen += taosEncodeVariantI32(buf, pCfg->tsdbId); - tlen += taosEncodeFixedI32(buf, pCfg->cacheBlockSize); - tlen += taosEncodeVariantI32(buf, pCfg->totalBlocks); - tlen += taosEncodeVariantI32(buf, pCfg->daysPerFile); - tlen += taosEncodeVariantI32(buf, pCfg->keep); - tlen += taosEncodeVariantI32(buf, pCfg->keep1); - tlen += taosEncodeVariantI32(buf, pCfg->keep2); - tlen += taosEncodeVariantI32(buf, pCfg->minRowsPerFileBlock); - tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); - tlen += taosEncodeFixedI8(buf, pCfg->precision); - tlen += taosEncodeFixedI8(buf, pCfg->compression); - tlen += taosEncodeFixedI8(buf, pCfg->update); - tlen += taosEncodeFixedI8(buf, pCfg->cacheLastRow); - - return tlen; -} - -static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { - buf = taosDecodeVariantI32(buf, &(pCfg->tsdbId)); - buf = taosDecodeFixedI32(buf, &(pCfg->cacheBlockSize)); - buf = taosDecodeVariantI32(buf, &(pCfg->totalBlocks)); - buf = taosDecodeVariantI32(buf, &(pCfg->daysPerFile)); - buf = taosDecodeVariantI32(buf, &(pCfg->keep)); - buf = taosDecodeVariantI32(buf, &(pCfg->keep1)); - buf = taosDecodeVariantI32(buf, &(pCfg->keep2)); - buf = taosDecodeVariantI32(buf, &(pCfg->minRowsPerFileBlock)); - buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); - buf = taosDecodeFixedI8(buf, &(pCfg->precision)); - buf = taosDecodeFixedI8(buf, &(pCfg->compression)); - buf = taosDecodeFixedI8(buf, &(pCfg->update)); - buf = taosDecodeFixedI8(buf, &(pCfg->cacheLastRow)); - - return buf; -} - -static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { - // TODO - // STsdbCache *pCache = pRepo->tsdbCache; - // int oldNumOfBlocks = pCache->totalCacheBlocks; - - // tsdbLockRepo((TsdbRepoT *)pRepo); - - // ASSERT(pCache->totalCacheBlocks != totalBlocks); - - // if (pCache->totalCacheBlocks < totalBlocks) { - // ASSERT(pCache->totalCacheBlocks == pCache->pool.numOfCacheBlocks); - // int blocksToAdd = pCache->totalCacheBlocks - totalBlocks; - // pCache->totalCacheBlocks = totalBlocks; - // for (int i = 0; i < blocksToAdd; i++) { - // if (tsdbAddCacheBlockToPool(pCache) < 0) { - // tsdbUnLockRepo((TsdbRepoT *)pRepo); - // tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId); - // return -1; - // } - // } - // } else { - // pCache->totalCacheBlocks = totalBlocks; - // tsdbAdjustCacheBlocks(pCache); - // } - // pRepo->config.totalBlocks = totalBlocks; - - // tsdbUnLockRepo((TsdbRepoT *)pRepo); - // tsdbDebug("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, - // totalBlocks); - return 0; -} - -#if 0 - -TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - - STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid); - if (pTable == NULL) return -1; - - return TSDB_GET_TABLE_LAST_KEY(pTable); -} - -#endif - -static void tsdbStartStream(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, - tsdbGetTableSchemaImpl(pTable, false, false, -1)); - } - } -} - - -static void tsdbStopStream(STsdbRepo *pRepo) { - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pMeta->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - (*pRepo->appH.cqDropFunc)(pTable->cqhandle); - } - } -} +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 07f001f68aecba589c68aa078fdb23dd071bbdd5..20ec426018a39e554fb03e9bb11399dbce1f3fcc 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -13,12 +13,23 @@ * along with this program. If not, see . */ -#include "tsdb.h" -#include "tsdbMain.h" +#include "tsdbint.h" #define TSDB_DATA_SKIPLIST_LEVEL 5 #define TSDB_MAX_INSERT_BATCH 512 +typedef struct { + int32_t totalLen; + int32_t len; + SDataRow row; +} SSubmitBlkIter; + +typedef struct { + int32_t totalLen; + int32_t len; + void * pMsg; +} SSubmitMsgIter; + static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); static void tsdbFreeMemTable(SMemTable *pMemTable); static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); @@ -41,8 +52,8 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey, TSKEY now); -int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { - STsdbRepo * pRepo = (STsdbRepo *)repo; +int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { + STsdbRepo * pRepo = repo; SSubmitMsgIter msgIter = {0}; SSubmitBlk * pBlock = NULL; int32_t affectedrows = 0; @@ -113,33 +124,80 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { return 0; } -int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { +int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot, SArray *pATable) { + memset(pSnapshot, 0, sizeof(*pSnapshot)); + if (tsdbLockRepo(pRepo) < 0) return -1; - *pMem = pRepo->mem; - *pIMem = pRepo->imem; - tsdbRefMemTable(pRepo, *pMem); - tsdbRefMemTable(pRepo, *pIMem); + pSnapshot->omem = pRepo->mem; + pSnapshot->imem = pRepo->imem; + tsdbRefMemTable(pRepo, pRepo->mem); + tsdbRefMemTable(pRepo, pRepo->imem); if (tsdbUnlockRepo(pRepo) < 0) return -1; - if (*pMem != NULL) taosRLockLatch(&((*pMem)->latch)); + if (pSnapshot->omem) { + taosRLockLatch(&(pSnapshot->omem->latch)); + + pSnapshot->mem = &(pSnapshot->mtable); + + pSnapshot->mem->tData = (STableData **)calloc(pSnapshot->omem->maxTables, sizeof(STableData *)); + if (pSnapshot->mem->tData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + taosRUnLockLatch(&(pSnapshot->omem->latch)); + tsdbUnRefMemTable(pRepo, pSnapshot->omem); + tsdbUnRefMemTable(pRepo, pSnapshot->imem); + pSnapshot->mem = NULL; + pSnapshot->imem = NULL; + pSnapshot->omem = NULL; + return -1; + } + + pSnapshot->mem->keyFirst = pSnapshot->omem->keyFirst; + pSnapshot->mem->keyLast = pSnapshot->omem->keyLast; + pSnapshot->mem->numOfRows = pSnapshot->omem->numOfRows; + pSnapshot->mem->maxTables = pSnapshot->omem->maxTables; + + for (size_t i = 0; i < taosArrayGetSize(pATable); i++) { + STable * pTable = *(STable **)taosArrayGet(pATable, i); + int32_t tid = TABLE_TID(pTable); + STableData *pTableData = (tid < pSnapshot->omem->maxTables) ? pSnapshot->omem->tData[tid] : NULL; - tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem); + if ((pTableData == NULL) || (TABLE_UID(pTable) != pTableData->uid)) continue; + + pSnapshot->mem->tData[tid] = pTableData; + T_REF_INC(pTableData); + } + + taosRUnLockLatch(&(pSnapshot->omem->latch)); + } + + tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem); return 0; } -void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) { - tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem); +void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemSnapshot *pSnapshot) { + tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pSnapshot->omem, pSnapshot->imem); - if (pMem != NULL) { - taosRUnLockLatch(&(pMem->latch)); - tsdbUnRefMemTable(pRepo, pMem); - } + if (pSnapshot->mem) { + ASSERT(pSnapshot->omem != NULL); + + for (size_t i = 0; i < pSnapshot->mem->maxTables; i++) { + STableData *pTableData = pSnapshot->mem->tData[i]; + if (pTableData) { + tsdbFreeTableData(pTableData); + } + } + tfree(pSnapshot->mem->tData); - if (pIMem != NULL) { - tsdbUnRefMemTable(pRepo, pIMem); + tsdbUnRefMemTable(pRepo, pSnapshot->omem); } + + tsdbUnRefMemTable(pRepo, pSnapshot->imem); + + pSnapshot->mem = NULL; + pSnapshot->imem = NULL; + pSnapshot->omem = NULL; } void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { @@ -205,11 +263,13 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { } int tsdbAsyncCommit(STsdbRepo *pRepo) { - if (pRepo->mem == NULL) return 0; - - sem_wait(&(pRepo->readyToCommit)); + tsem_wait(&(pRepo->readyToCommit)); ASSERT(pRepo->imem == NULL); + if (pRepo->mem == NULL) { + tsem_post(&(pRepo->readyToCommit)); + return 0; + } if (pRepo->code != TSDB_CODE_SUCCESS) { tsdbWarn("vgId:%d try to commit when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); @@ -225,12 +285,12 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { return 0; } -int tsdbSyncCommit(TSDB_REPO_T *repo) { - STsdbRepo *pRepo = (STsdbRepo *)repo; +int tsdbSyncCommit(STsdbRepo *repo) { + STsdbRepo *pRepo = repo; tsdbAsyncCommit(pRepo); - sem_wait(&(pRepo->readyToCommit)); - sem_post(&(pRepo->readyToCommit)); + tsem_wait(&(pRepo->readyToCommit)); + tsem_post(&(pRepo->readyToCommit)); if (pRepo->code != TSDB_CODE_SUCCESS) { terrno = pRepo->code; @@ -254,14 +314,17 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) { */ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { - ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL); + ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; - STSchema *pSchema = NULL; - TSKEY rowKey = 0; - TSKEY fKey = 0; - bool isRowDel = false; - int filterIter = 0; - SDataRow row = NULL; + STSchema * pSchema = NULL; + TSKEY rowKey = 0; + TSKEY fKey = 0; + bool isRowDel = false; + int filterIter = 0; + SDataRow row = NULL; + SMergeInfo mInfo; + + if (pMergeInfo == NULL) pMergeInfo = &mInfo; memset(pMergeInfo, 0, sizeof(*pMergeInfo)); pMergeInfo->keyFirst = INT64_MAX; @@ -420,7 +483,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { STableData *pTableData = (STableData *)calloc(1, sizeof(*pTableData)); if (pTableData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + return NULL; } pTableData->uid = TABLE_UID(pTable); @@ -433,30 +496,27 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; + free(pTableData); + return NULL; } - return pTableData; + T_REF_INC(pTableData); -_err: - tsdbFreeTableData(pTableData); - return NULL; + return pTableData; } static void tsdbFreeTableData(STableData *pTableData) { if (pTableData) { - tSkipListDestroy(pTableData->pData); - free(pTableData); + int32_t ref = T_REF_DEC(pTableData); + if (ref == 0) { + tSkipListDestroy(pTableData->pData); + free(pTableData); + } } } static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); } -void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) { - *minKey = fileId * daysPerFile * tsMsPerDay[precision]; - *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; -} - static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { ASSERT(pMemTable->maxTables < maxTables); @@ -927,4 +987,4 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow } return 0; -} \ No newline at end of file +} diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 7b08178f4949eff0c4e8802f1d04cf1c85554c5b..824f69608e8c9d2d00e3219d54b484304827e8ae 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -12,23 +12,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include -#include "hash.h" -#include "taosdef.h" -#include "tchecksum.h" -#include "tsdb.h" -#include "tsdbMain.h" -#include "tskiplist.h" +#include "tsdbint.h" #define TSDB_SUPER_TABLE_SL_LEVEL 5 #define DEFAULT_TAG_INDEX_COLUMN 0 static int tsdbCompareSchemaVersion(const void *key1, const void *key2); -static int tsdbRestoreTable(void *pHandle, void *cont, int contLen); -static void tsdbOrgMeta(void *pHandle); static char * getTagIndexKey(const void *pData); static STable *tsdbNewTable(); -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper); +static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable); static void tsdbFreeTable(STable *pTable); static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock); static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock); @@ -51,14 +43,16 @@ static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STabl static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); +static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema); // ------------------ OUTER FUNCTIONS ------------------ -int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { +int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; STable * super = NULL; STable * table = NULL; - int newSuper = 0; + bool newSuper = false; + bool superChanged = false; int tid = pCfg->tableId.tid; STable * pTable = NULL; @@ -93,18 +87,29 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { if (pCfg->type == TSDB_CHILD_TABLE) { super = tsdbGetTableByUid(pMeta, pCfg->superUid); if (super == NULL) { // super table not exists, try to create it - newSuper = 1; - super = tsdbCreateTableFromCfg(pCfg, true); + newSuper = true; + super = tsdbCreateTableFromCfg(pCfg, true, NULL); if (super == NULL) goto _err; } else { if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) { terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; goto _err; } + + if (schemaVersion(pCfg->tagSchema) > schemaVersion(super->tagSchema)) { + // tag schema out of date, need to update super table tag version + STSchema *pOldSchema = super->tagSchema; + TSDB_WLOCK_TABLE(super); + super->tagSchema = tdDupSchema(pCfg->tagSchema); + TSDB_WUNLOCK_TABLE(super); + tdFreeSchema(pOldSchema); + + superChanged = true; + } } } - table = tsdbCreateTableFromCfg(pCfg, false); + table = tsdbCreateTableFromCfg(pCfg, false, super); if (table == NULL) goto _err; // Register to meta @@ -125,7 +130,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { // TODO: refactor duplicate codes int tlen = 0; void *pBuf = NULL; - if (newSuper) { + if (newSuper || superChanged) { tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, super); pBuf = tsdbAllocBytes(pRepo, tlen); if (pBuf == NULL) goto _err; @@ -148,7 +153,7 @@ _err: return -1; } -int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { +int tsdbDropTable(STsdbRepo *repo, STableId tableId) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; uint64_t uid = tableId.uid; @@ -208,9 +213,9 @@ void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_ char *val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId); assert(type == pCol->type && bytes == pCol->bytes); - if (val != NULL && IS_VAR_DATA_TYPE(type)) { - assert(varDataLen(val) < pCol->bytes); - } + // if (val != NULL && IS_VAR_DATA_TYPE(type)) { + // assert(varDataLen(val) < pCol->bytes); + // } return val; } @@ -253,7 +258,7 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { } } if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetName(pCfg, pMsg->tableId, true) < 0) goto _err; + if (tsdbTableSetName(pCfg, pMsg->tableFname, true) < 0) goto _err; if (numOfTags > 0) { // Decode tag schema @@ -265,7 +270,7 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { } } if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err; + if (tsdbTableSetSName(pCfg, pMsg->stableFname, true) < 0) goto _err; if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err; int32_t tagDataLen = htonl(pMsg->tagDataLen); @@ -301,7 +306,7 @@ static UNUSED_FUNC int32_t colIdCompar(const void* left, const void* right) { return (colId < p2->colId)? -1:1; } -int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { +int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; STSchema * pNewSchema = NULL; @@ -469,6 +474,8 @@ void tsdbFreeMeta(STsdbMeta *pMeta) { } int tsdbOpenMeta(STsdbRepo *pRepo) { + return 0; +#if 0 char * fname = NULL; STsdbMeta *pMeta = pRepo->tsdbMeta; ASSERT(pMeta != NULL); @@ -479,11 +486,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) { goto _err; } - pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo); - if (pMeta->pStore == NULL) { - tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } + // pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo); + // if (pMeta->pStore == NULL) { + // tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno)); + // goto _err; + // } tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo)); tfree(fname); @@ -492,6 +499,7 @@ int tsdbOpenMeta(STsdbRepo *pRepo) { _err: tfree(fname); return -1; +#endif } int tsdbCloseMeta(STsdbRepo *pRepo) { @@ -500,7 +508,7 @@ int tsdbCloseMeta(STsdbRepo *pRepo) { STable * pTable = NULL; if (pMeta == NULL) return 0; - tdCloseKVStore(pMeta->pStore); + // tdCloseKVStore(pMeta->pStore); for (int i = 1; i < pMeta->maxTables; i++) { tsdbFreeTable(pMeta->tables[i]); } @@ -567,12 +575,13 @@ void tsdbRefTable(STable *pTable) { } void tsdbUnRefTable(STable *pTable) { - int32_t ref = T_REF_DEC(pTable); - tsdbDebug("unref table %s uid:%"PRIu64" tid:%d, refCount:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), ref); + uint64_t uid = TABLE_UID(pTable); + int32_t tid = TABLE_TID(pTable); + int32_t ref = T_REF_DEC(pTable); - if (ref == 0) { - // tsdbDebug("destory table name:%s uid:%"PRIu64", tid:%d", TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable)); + tsdbDebug("unref table, uid:%" PRIu64 " tid:%d, refCount:%d", uid, tid, ref); + if (ref == 0) { if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { tsdbUnRefTable(pTable->pSuper); } @@ -609,10 +618,8 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, } } -// ------------------ LOCAL FUNCTIONS ------------------ -static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { - STsdbRepo *pRepo = (STsdbRepo *)pHandle; - STable * pTable = NULL; +int tsdbRestoreTable(STsdbRepo *pRepo, void *cont, int contLen) { + STable *pTable = NULL; if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; @@ -631,8 +638,7 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { return 0; } -static void tsdbOrgMeta(void *pHandle) { - STsdbRepo *pRepo = (STsdbRepo *)pHandle; +void tsdbOrgMeta(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; for (int i = 1; i < pMeta->maxTables; i++) { @@ -643,6 +649,7 @@ static void tsdbOrgMeta(void *pHandle) { } } +// ------------------ LOCAL FUNCTIONS ------------------ static char *getTagIndexKey(const void *pData) { STable *pTable = (STable *)pData; @@ -668,7 +675,7 @@ static STable *tsdbNewTable() { return pTable; } -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { +static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable) { STable *pTable = NULL; size_t tsize = 0; @@ -720,6 +727,9 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { if (pCfg->type == TSDB_CHILD_TABLE) { TABLE_SUID(pTable) = pCfg->superUid; + if (tsdbCheckTableTagVal(pCfg->tagValues, pSTable->tagSchema) < 0) { + goto _err; + } pTable->tagVal = tdKVRowDup(pCfg->tagValues); if (pTable->tagVal == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1296,3 +1306,20 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { return 0; } + +static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema) { + for (size_t i = 0; i < kvRowNCols(pKVRow); i++) { + SColIdx * pColIdx = kvRowColIdxAt(pKVRow, i); + STColumn *pCol = tdGetColOfID(pSchema, pColIdx->colId); + + if ((pCol == NULL) || (!IS_VAR_DATA_TYPE(pCol->type))) continue; + + void *pValue = tdGetKVRowValOfCol(pKVRow, pCol->colId); + if (varDataTLen(pValue) > pCol->bytes) { + terrno = TSDB_CODE_TDB_IVLD_TAG_VAL; + return -1; + } + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c deleted file mode 100644 index b53b8ed5b419151380cb79a88907f2aab64073ea..0000000000000000000000000000000000000000 --- a/src/tsdb/src/tsdbRWHelper.c +++ /dev/null @@ -1,1753 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#define TAOS_RANDOM_FILE_FAIL_TEST -#include "os.h" -#include "talgo.h" -#include "tchecksum.h" -#include "tcoding.h" -#include "tscompression.h" -#include "tsdbMain.h" - -#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) -#define TSDB_KEY_COL_OFFSET 0 -#define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock)) -#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last) - -static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); -static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock, - bool isLast, bool isSuperBlock); -static int compareKeyBlock(const void *arg1, const void *arg2); -static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize); -static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo); -static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static void tsdbResetHelperFileImpl(SRWHelper *pHelper); -static int tsdbInitHelperFile(SRWHelper *pHelper); -static void tsdbDestroyHelperFile(SRWHelper *pHelper); -static void tsdbResetHelperTableImpl(SRWHelper *pHelper); -static void tsdbResetHelperTable(SRWHelper *pHelper); -static void tsdbInitHelperTable(SRWHelper *pHelper); -static void tsdbDestroyHelperTable(SRWHelper *pHelper); -static void tsdbResetHelperBlockImpl(SRWHelper *pHelper); -static void tsdbResetHelperBlock(SRWHelper *pHelper); -static int tsdbInitHelperBlock(SRWHelper *pHelper); -static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type); -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, - int maxPoints, char *buffer, int bufferSize); -static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds, - int numOfColIds); -static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols); -static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx); -static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); -static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey); -static void tsdbDestroyHelperBlock(SRWHelper *pHelper); -static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBlock, SCompCol *pCompCol, - SDataCol *pDataCol); -static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock); -static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, - int *blkIdx); -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update); -static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps); -static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx); - -// ---------------------- INTERNAL FUNCTIONS ---------------------- -int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { - return tsdbInitHelper(pHelper, pRepo, TSDB_READ_HELPER); -} - -int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { - return tsdbInitHelper(pHelper, pRepo, TSDB_WRITE_HELPER); -} - -void tsdbDestroyHelper(SRWHelper *pHelper) { - if (pHelper) { - taosTZfree(pHelper->pBuffer); - taosTZfree(pHelper->compBuffer); - tsdbDestroyHelperFile(pHelper); - tsdbDestroyHelperTable(pHelper); - tsdbDestroyHelperBlock(pHelper); - memset((void *)pHelper, 0, sizeof(*pHelper)); - } -} - -void tsdbResetHelper(SRWHelper *pHelper) { - if (pHelper) { - // Reset the block part - tsdbResetHelperBlockImpl(pHelper); - - // Reset the table part - tsdbResetHelperTableImpl(pHelper); - - // Reset the file part - tsdbCloseHelperFile(pHelper, false, NULL); - tsdbResetHelperFileImpl(pHelper); - - pHelper->state = TSDB_HELPER_CLEAR_STATE; - } -} - -int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { - ASSERT(pHelper != NULL && pGroup != NULL); - SFile * pFile = NULL; - STsdbRepo *pRepo = pHelper->pRepo; - - // Clear the helper object - tsdbResetHelper(pHelper); - - ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE); - - // Set the files - pHelper->files.fGroup = *pGroup; - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), pGroup->fileId, TSDB_FILE_TYPE_NHEAD, - helperNewHeadF(pHelper)->fname); - tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), pGroup->fileId, TSDB_FILE_TYPE_NLAST, - helperNewLastF(pHelper)->fname); - } - - // Open the files - if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) return -1; - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; - if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; - - // Create and open .h - pFile = helperNewHeadF(pHelper); - if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; - if (tsdbUpdateFileHeader(pFile) < 0) return -1; - - // Create and open .l file if should - if (tsdbShouldCreateNewLast(pHelper)) { - pFile = helperNewLastF(pHelper); - if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; - pFile->info.len = 0; - if (tsdbUpdateFileHeader(pFile) < 0) return -1; - } - } else { - if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) return -1; - if (tsdbOpenFile(helperLastF(pHelper), O_RDONLY) < 0) return -1; - } - - helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN); - - return 0; -} - -int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError, SFileGroup *pGroup) { - SFile *pFile = NULL; - - pFile = helperHeadF(pHelper); - tsdbCloseFile(pFile); - - pFile = helperDataF(pHelper); - if (pFile->fd > 0) { - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - } else { - ASSERT(pGroup != NULL); - taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_DATA].info.size); - } - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - } - - pFile = helperLastF(pHelper); - if (pFile->fd > 0) { - if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - } else { - ASSERT(pGroup != NULL); - taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_LAST].info.size); - } - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - } - - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - pFile = helperNewHeadF(pHelper); - if (pFile->fd > 0) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - if (hasError) (void)remove(pFile->fname); - } - - pFile = helperNewLastF(pHelper); - if (pFile->fd > 0) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - if (hasError) (void)remove(pFile->fname); - } - } - return 0; -} - -int tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { - ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)); - - // Clear members and state used by previous table - tsdbResetHelperTable(pHelper); - ASSERT(helperHasState(pHelper, (TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD))); - - pHelper->tableInfo.tid = pTable->tableId.tid; - pHelper->tableInfo.uid = pTable->tableId.uid; - STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - - if (tdInitDataCols(pHelper->pDataCols[0], pSchema) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (tdInitDataCols(pHelper->pDataCols[1], pSchema) < 0) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (pHelper->idxH.numOfIdx > 0) { - while (true) { - if (pHelper->idxH.curIdx >= pHelper->idxH.numOfIdx) { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - break; - } - - SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); - if (pIdx->tid == TABLE_TID(pTable)) { - if (pIdx->uid == TABLE_UID(pTable)) { - pHelper->curCompIdx = *pIdx; - } else { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - } - pHelper->idxH.curIdx++; - break; - } else if (pIdx->tid > TABLE_TID(pTable)) { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - break; - } else { - pHelper->idxH.curIdx++; - } - } - } else { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - } - - if (helperType(pHelper) == TSDB_WRITE_HELPER && pHelper->curCompIdx.hasLast) { - pHelper->hasOldLastBlock = true; - } - - helperSetState(pHelper, TSDB_HELPER_TABLE_SET); - ASSERT(pHelper->state == ((TSDB_HELPER_TABLE_SET << 1) - 1)); - - return 0; -} - -int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - - SCompIdx *pIdx = &(pHelper->curCompIdx); - int blkIdx = 0; - - ASSERT(pIdx->offset == 0 || pIdx->uid == TABLE_UID(pCommitIter->pTable)); - if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; - - while (true) { - ASSERT(blkIdx <= (int)pIdx->numOfBlocks); - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over - - if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) { - if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1; - blkIdx = pIdx->numOfBlocks; - } else { - if (tsdbProcessMergeCommit(pHelper, pCommitIter, pDataCols, maxKey, &blkIdx) < 0) return -1; - } - } - - return 0; -} - -int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { - STsdbCfg *pCfg = &pHelper->pRepo->config; - - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - SCompIdx * pIdx = &(pHelper->curCompIdx); - SCompBlock compBlock = {0}; - if (TSDB_NLAST_FILE_OPENED(pHelper) && (pHelper->hasOldLastBlock)) { - if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; - - SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); - ASSERT(pCompBlock->last); - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && - pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); - if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) - return -1; - - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; - -#if 0 - if (pCompBlock->numOfSubBlocks > 1) { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && - pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); - if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) - return -1; - - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; - } else { - if (lseek(helperLastF(pHelper)->fd, pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperLastF(pHelper)->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pCompBlock->offset = lseek(helperNewLastF(pHelper)->fd, 0, SEEK_END); - if (pCompBlock->offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewLastF(pHelper)->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { - tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo), - helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } -#endif - - pHelper->hasOldLastBlock = false; - } - - return 0; -} - -int tsdbWriteCompInfo(SRWHelper *pHelper) { - SCompIdx *pIdx = &(pHelper->curCompIdx); - off_t offset = 0; - SFile * pFile = helperNewHeadF(pHelper); - - if (pIdx->len > 0) { - if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; - } else { - pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; - pHelper->pCompInfo->uid = pHelper->tableInfo.uid; - pHelper->pCompInfo->tid = pHelper->tableInfo.tid; - ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && - (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); - taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - } - - pFile->info.magic = taosCalcChecksum( - pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pIdx->offset = offset; - pIdx->uid = pHelper->tableInfo.uid; - pIdx->tid = pHelper->tableInfo.tid; - ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - - if (taosWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosTSizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { - pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, taosTSizeof(pHelper->pWIdx) == 0 ? 1024 : taosTSizeof(pHelper->pWIdx) * 2); - if (pHelper->pWIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - - void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); - pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); - - pFile->info.size += pIdx->len; - // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); - } - - return 0; -} - -int tsdbWriteCompIdx(SRWHelper *pHelper) { - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - off_t offset = 0; - - SFile *pFile = helperNewHeadF(pHelper); - - pFile->info.len += sizeof(TSCKSUM); - if (taosTSizeof(pHelper->pWIdx) < pFile->info.len) { - pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, pFile->info.len); - if (pHelper->pWIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); - pFile->info.magic = taosCalcChecksum( - pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pWIdx, pFile->info.len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); - - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - ASSERT(offset == pFile->info.size); - - if (taosWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pFile->info.offset = offset; - pFile->info.size += pFile->info.len; - // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); - - return 0; -} - -int tsdbLoadCompIdxImpl(SFile *pFile, uint32_t offset, uint32_t len, void *buffer) { - const char *prefixMsg = "failed to load SCompIdx part"; - if (lseek(pFile->fd, offset, SEEK_SET) < 0) { - tsdbError("%s: seek to file %s offset %u failed since %s", prefixMsg, pFile->fname, offset, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosRead(pFile->fd, buffer, len) < len) { - tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, offset, len, - strerror(errno)); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)buffer, len)) { - tsdbError("%s: file %s corrupted, offset %u len %u", prefixMsg, pFile->fname, offset, len); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - return 0; -} - -int tsdbDecodeSCompIdxImpl(void *buffer, uint32_t len, SCompIdx **ppCompIdx, int *numOfIdx) { - int nIdx = 0; - void *pPtr = buffer; - - while (POINTER_DISTANCE(pPtr, buffer) < (int)(len - sizeof(TSCKSUM))) { - size_t tlen = taosTSizeof(*ppCompIdx); - if (tlen < sizeof(SCompIdx) * (nIdx + 1)) { - *ppCompIdx = (SCompIdx *)taosTRealloc(*ppCompIdx, (tlen == 0) ? 1024 : tlen * 2); - if (*ppCompIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - - pPtr = tsdbDecodeSCompIdx(pPtr, &((*ppCompIdx)[nIdx])); - if (pPtr == NULL) { - tsdbError("failed to decode SCompIdx part, idx:%d", nIdx); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - nIdx++; - - ASSERT(nIdx == 1 || (*ppCompIdx)[nIdx - 1].tid > (*ppCompIdx)[nIdx - 2].tid); - ASSERT(POINTER_DISTANCE(pPtr, buffer) <= (int)(len - sizeof(TSCKSUM))); - } - - *numOfIdx = nIdx; - return 0; -} - -int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { - ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); - SFile *pFile = helperHeadF(pHelper); - - if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { - // If not load from file, just load it in object - if (pFile->info.len > 0) { - if ((pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - // Load SCompIdx binary from file - if (tsdbLoadCompIdxImpl(pFile, pFile->info.offset, pFile->info.len, (void *)(pHelper->pBuffer)) < 0) { - return -1; - } - - // Decode the SCompIdx part - if (tsdbDecodeSCompIdxImpl(pHelper->pBuffer, pFile->info.len, &(pHelper->idxH.pIdxArray), - &(pHelper->idxH.numOfIdx)) < 0) { - tsdbError("vgId:%d failed to decode SCompIdx part from file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, - tstrerror(errno)); - return -1; - } - } - } - helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); - - // Copy the memory for outside usage - if (target && pHelper->idxH.numOfIdx > 0) - memcpy(target, pHelper->idxH.pIdxArray, sizeof(SCompIdx) * pHelper->idxH.numOfIdx); - - return 0; -} - -int tsdbLoadCompInfoImpl(SFile *pFile, SCompIdx *pIdx, SCompInfo **ppCompInfo) { - const char *prefixMsg = "failed to load SCompInfo/SCompBlock part"; - - if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) { - tsdbError("%s: seek to file %s offset %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - *ppCompInfo = taosTRealloc((void *)(*ppCompInfo), pIdx->len); - if (*ppCompInfo == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (taosRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) { - tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, pIdx->len, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)(*ppCompInfo), pIdx->len)) { - tsdbError("%s: file %s corrupted, offset %u len %u", prefixMsg, pFile->fname, pIdx->offset, pIdx->len); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - return 0; -} - -int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { - ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - - SCompIdx *pIdx = &(pHelper->curCompIdx); - - SFile *pFile = helperHeadF(pHelper); - - if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (pIdx->offset > 0) { - ASSERT(pIdx->uid == pHelper->tableInfo.uid); - - if (tsdbLoadCompInfoImpl(pFile, pIdx, &(pHelper->pCompInfo)) < 0) return -1; - - ASSERT(pIdx->uid == pHelper->pCompInfo->uid && pIdx->tid == pHelper->pCompInfo->tid); - } - - helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); - } - - if (target) memcpy(target, (void *)(pHelper->pCompInfo), pIdx->len); - - return 0; -} - -int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { - ASSERT(pCompBlock->numOfSubBlocks <= 1); - SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); - - if (lseek(pFile->fd, (off_t)pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - size_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols); - pHelper->pCompData = taosTRealloc((void *)pHelper->pCompData, tsize); - if (pHelper->pCompData == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - if (taosRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) { - tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompData, (uint32_t)tsize)) { - tsdbError("vgId:%d file %s is broken, offset %" PRId64 " size %" PRIzu "", REPO_ID(pHelper->pRepo), pFile->fname, - (int64_t)pCompBlock->offset, tsize); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - ASSERT(pCompBlock->numOfCols == pHelper->pCompData->numOfCols); - - if (target) memcpy(target, pHelper->pCompData, tsize); - - return 0; -} - -void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) { - SCompData *pCompData = pHelper->pCompData; - - for (int i = 0, j = 0; i < numOfCols;) { - if (j >= pCompData->numOfCols) { - pStatis[i].numOfNull = -1; - i++; - continue; - } - - if (pStatis[i].colId == pCompData->cols[j].colId) { - pStatis[i].sum = pCompData->cols[j].sum; - pStatis[i].max = pCompData->cols[j].max; - pStatis[i].min = pCompData->cols[j].min; - pStatis[i].maxIndex = pCompData->cols[j].maxIndex; - pStatis[i].minIndex = pCompData->cols[j].minIndex; - pStatis[i].numOfNull = pCompData->cols[j].numOfNull; - i++; - j++; - } else if (pStatis[i].colId < pCompData->cols[j].colId) { - pStatis[i].numOfNull = -1; - i++; - } else { - j++; - } - } -} - -int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo, int16_t *colIds, int numOfColIds) { - ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block - SCompBlock *pTCompBlock = pCompBlock; - - int numOfSubBlocks = pCompBlock->numOfSubBlocks; - if (numOfSubBlocks > 1) - pTCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset); - - tdResetDataCols(pHelper->pDataCols[0]); - if (tsdbLoadBlockDataColsImpl(pHelper, pTCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err; - for (int i = 1; i < numOfSubBlocks; i++) { - tdResetDataCols(pHelper->pDataCols[1]); - pTCompBlock++; - if (tsdbLoadBlockDataColsImpl(pHelper, pTCompBlock, pHelper->pDataCols[1], colIds, numOfColIds) < 0) goto _err; - if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err; - } - - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && - dataColsKeyFirst(pHelper->pDataCols[0]) == pCompBlock->keyFirst && - dataColsKeyLast(pHelper->pDataCols[0]) == pCompBlock->keyLast); - - return 0; - -_err: - return -1; -} - -int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo) { - SCompBlock *pTCompBlock = pCompBlock; - - int numOfSubBlock = pCompBlock->numOfSubBlocks; - if (numOfSubBlock > 1) - pTCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset); - - tdResetDataCols(pHelper->pDataCols[0]); - if (tsdbLoadBlockDataImpl(pHelper, pTCompBlock, pHelper->pDataCols[0]) < 0) goto _err; - for (int i = 1; i < numOfSubBlock; i++) { - tdResetDataCols(pHelper->pDataCols[1]); - pTCompBlock++; - if (tsdbLoadBlockDataImpl(pHelper, pTCompBlock, pHelper->pDataCols[1]) < 0) goto _err; - if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err; - } - - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && - dataColsKeyFirst(pHelper->pDataCols[0]) == pCompBlock->keyFirst && - dataColsKeyLast(pHelper->pDataCols[0]) == pCompBlock->keyLast); - - return 0; - -_err: - return -1; -} - -// ---------------------- INTERNAL FUNCTIONS ---------------------- -static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { - ASSERT(helperLastF(pHelper)->fd > 0); - struct stat st; - if (fstat(helperLastF(pHelper)->fd, &st) < 0) return true; - if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true; - return false; -} - -static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock, - bool isLast, bool isSuperBlock) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - SCompData *pCompData = (SCompData *)(pHelper->pBuffer); - int64_t offset = 0; - int rowsToWrite = pDataCols->numOfRows; - - ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock); - ASSERT(isLast ? rowsToWrite < pCfg->minRowsPerFileBlock : true); - - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - int nColsNotAllNull = 0; - for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol *pDataCol = pDataCols->cols + ncol; - SCompCol *pCompCol = pCompData->cols + nColsNotAllNull; - - if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it - continue; - } - - memset(pCompCol, 0, sizeof(*pCompCol)); - - pCompCol->colId = pDataCol->colId; - pCompCol->type = pDataCol->type; - if (tDataTypeDesc[pDataCol->type].getStatisFunc) { - (*tDataTypeDesc[pDataCol->type].getStatisFunc)( - pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max), &(pCompCol->sum), &(pCompCol->minIndex), - &(pCompCol->maxIndex), &(pCompCol->numOfNull)); - } - nColsNotAllNull++; - } - - ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols); - - // Compress the data if neccessary - int tcol = 0; - int32_t toffset = 0; - int32_t tsize = TSDB_GET_COMPCOL_LEN(nColsNotAllNull); - int32_t lsize = tsize; - int32_t keyLen = 0; - for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { - if (ncol != 0 && tcol >= nColsNotAllNull) break; - - SDataCol *pDataCol = pDataCols->cols + ncol; - SCompCol *pCompCol = pCompData->cols + tcol; - - if (ncol != 0 && (pDataCol->colId != pCompCol->colId)) continue; - void *tptr = POINTER_SHIFT(pCompData, lsize); - - int32_t flen = 0; // final length - int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); - - if (pCfg->compression) { - if (pCfg->compression == TWO_STAGE_COMP) { - pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES); - if (pHelper->compBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - - flen = (*(tDataTypeDesc[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, - (int32_t)taosTSizeof(pHelper->pBuffer) - lsize, pCfg->compression, - pHelper->compBuffer, (int32_t)taosTSizeof(pHelper->compBuffer)); - } else { - flen = tlen; - memcpy(tptr, pDataCol->pData, flen); - } - - // Add checksum - ASSERT(flen > 0); - flen += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); - pFile->info.magic = - taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); - - if (ncol != 0) { - pCompCol->offset = toffset; - pCompCol->len = flen; - tcol++; - } else { - keyLen = flen; - } - - toffset += flen; - lsize += flen; - } - - pCompData->delimiter = TSDB_FILE_DELIMITER; - pCompData->uid = pHelper->tableInfo.uid; - pCompData->numOfCols = nColsNotAllNull; - - taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize); - pFile->info.magic = taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(pCompData, tsize - sizeof(TSCKSUM)), - sizeof(TSCKSUM)); - - // Write the whole block to file - if (taosWrite(pFile->fd, (void *)pCompData, lsize) < lsize) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // Update pCompBlock membership vairables - pCompBlock->last = isLast; - pCompBlock->offset = offset; - pCompBlock->algorithm = pCfg->compression; - pCompBlock->numOfRows = rowsToWrite; - pCompBlock->len = lsize; - pCompBlock->keyLen = keyLen; - pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0; - pCompBlock->numOfCols = nColsNotAllNull; - pCompBlock->keyFirst = dataColsKeyFirst(pDataCols); - pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1); - - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 - " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, - REPO_ID(helperRepo(pHelper)), pHelper->tableInfo.tid, pFile->fname, (int64_t)(pCompBlock->offset), - (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, - pCompBlock->keyLast); - - pFile->info.size += pCompBlock->len; - // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); - - return 0; - -_err: - return -1; -} - -static int compareKeyBlock(const void *arg1, const void *arg2) { - TSKEY key = *(TSKEY *)arg1; - SCompBlock *pBlock = (SCompBlock *)arg2; - - if (key < pBlock->keyFirst) { - return -1; - } else if (key > pBlock->keyLast) { - return 1; - } - - return 0; -} - -static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { - if (taosTSizeof((void *)pHelper->pCompInfo) <= esize) { - size_t tsize = esize + sizeof(SCompBlock) * 16; - pHelper->pCompInfo = (SCompInfo *)taosTRealloc(pHelper->pCompInfo, tsize); - if (pHelper->pCompInfo == NULL) return -1; - } - - return 0; -} - -static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { - SCompIdx *pIdx = &(pHelper->curCompIdx); - - ASSERT(blkIdx >= 0 && blkIdx <= (int)pIdx->numOfBlocks); - ASSERT(pCompBlock->numOfSubBlocks == 1); - - // Adjust memory if no more room - if (pIdx->len == 0) pIdx->len = sizeof(SCompInfo) + sizeof(TSCKSUM); - if (tsdbAdjustInfoSizeIfNeeded(pHelper, pIdx->len + sizeof(SCompInfo)) < 0) goto _err; - - // Change the offset - for (uint32_t i = 0; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); - } - - // Memmove if needed - int tsize = pIdx->len - (sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx); - if (tsize > 0) { - ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) < taosTSizeof(pHelper->pCompInfo)); - ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) + tsize <= taosTSizeof(pHelper->pCompInfo)); - memmove(POINTER_SHIFT(pHelper->pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1)), - POINTER_SHIFT(pHelper->pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx), tsize); - } - pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock; - - pIdx->numOfBlocks++; - pIdx->len += sizeof(SCompBlock); - ASSERT(pIdx->len <= taosTSizeof(pHelper->pCompInfo)); - pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast; - pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; - - if (pIdx->numOfBlocks > 1) { - ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst); - } - - ASSERT((blkIdx == pIdx->numOfBlocks -1) || (!pCompBlock->last)); - - tsdbDebug("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, - blkIdx); - - return 0; - -_err: - return -1; -} - -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) { - ASSERT(pCompBlock->numOfSubBlocks == 0); - - SCompIdx *pIdx = &(pHelper->curCompIdx); - ASSERT(blkIdx >= 0 && blkIdx < (int)pIdx->numOfBlocks); - - SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; - ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS); - - size_t spaceNeeded = - (pSCompBlock->numOfSubBlocks == 1) ? pIdx->len + sizeof(SCompBlock) * 2 : pIdx->len + sizeof(SCompBlock); - if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err; - - pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; - - // Add the sub-block - if (pSCompBlock->numOfSubBlocks > 1) { - size_t tsize = (size_t)(pIdx->len - (pSCompBlock->offset + pSCompBlock->len)); - if (tsize > 0) { - memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)), - (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); - - for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); - } - } - - *(SCompBlock *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len) = *pCompBlock; - - pSCompBlock->numOfSubBlocks++; - ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); - pSCompBlock->len += sizeof(SCompBlock); - pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; - pSCompBlock->keyFirst = pMergeInfo->keyFirst; - pSCompBlock->keyLast = pMergeInfo->keyLast; - pIdx->len += sizeof(SCompBlock); - } else { // Need to create two sub-blocks - void *ptr = NULL; - for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; - if (pTCompBlock->numOfSubBlocks > 1) { - ptr = POINTER_SHIFT(pHelper->pCompInfo, pTCompBlock->offset); - break; - } - } - - if (ptr == NULL) ptr = POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)); - - size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo)); - if (tsize > 0) { - memmove(POINTER_SHIFT(ptr, sizeof(SCompBlock) * 2), ptr, tsize); - for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2); - } - } - - ((SCompBlock *)ptr)[0] = *pSCompBlock; - ((SCompBlock *)ptr)[0].numOfSubBlocks = 0; - - ((SCompBlock *)ptr)[1] = *pCompBlock; - - pSCompBlock->numOfSubBlocks = 2; - pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; - pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo); - pSCompBlock->len = sizeof(SCompBlock) * 2; - pSCompBlock->keyFirst = pMergeInfo->keyFirst; - pSCompBlock->keyLast = pMergeInfo->keyLast; - - pIdx->len += (sizeof(SCompBlock) * 2); - } - - pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; - pIdx->hasLast = (uint32_t)pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; - - tsdbDebug("vgId:%d tid:%d a subblock is added at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx); - - return 0; - -_err: - return -1; -} - -static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { - ASSERT(pCompBlock->numOfSubBlocks == 1); - - SCompIdx *pIdx = &(pHelper->curCompIdx); - - ASSERT(blkIdx >= 0 && blkIdx < (int)pIdx->numOfBlocks); - - SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; - - ASSERT(pSCompBlock->numOfSubBlocks >= 1); - - // Delete the sub blocks it has - if (pSCompBlock->numOfSubBlocks > 1) { - size_t tsize = (size_t)(pIdx->len - (pSCompBlock->offset + pSCompBlock->len)); - if (tsize > 0) { - memmove(POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset), - POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset + pSCompBlock->len), tsize); - } - - for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { - SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; - if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); - } - - pIdx->len -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); - } - - *pSCompBlock = *pCompBlock; - - pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast; - pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; - - ASSERT((blkIdx == pIdx->numOfBlocks-1) || (!pCompBlock->last)); - - tsdbDebug("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, - blkIdx); - - return 0; -} - -static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { - SCompIdx *pCompIdx = &(pHelper->curCompIdx); - - ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks); - - SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx); - SCompBlock compBlock = *pCompBlock; - ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); - - if (pCompIdx->numOfBlocks == 1) { - memset(pCompIdx, 0, sizeof(*pCompIdx)); - } else { - int tsize = 0; - - if (compBlock.numOfSubBlocks > 1) { - tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks)); - - ASSERT(tsize > 0); - memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset), - POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks), - tsize); - - pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks; - } - - tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo)); - ASSERT(tsize > 0); - memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize); - - pCompIdx->len -= sizeof(SCompBlock); - - pCompIdx->numOfBlocks--; - pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last); - pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast; - } - - return 0; -} - -static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { - pHelper->idxH.numOfIdx = 0; - pHelper->idxH.curIdx = 0; - memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - helperHeadF(pHelper)->fd = -1; - helperDataF(pHelper)->fd = -1; - helperLastF(pHelper)->fd = -1; - helperNewHeadF(pHelper)->fd = -1; - helperNewLastF(pHelper)->fd = -1; -} - -static int tsdbInitHelperFile(SRWHelper *pHelper) { - tsdbResetHelperFileImpl(pHelper); - return 0; -} - -static void tsdbDestroyHelperFile(SRWHelper *pHelper) { - tsdbCloseHelperFile(pHelper, false, NULL); - tsdbResetHelperFileImpl(pHelper); - taosTZfree(pHelper->idxH.pIdxArray); - taosTZfree(pHelper->pWIdx); -} - -// ---------- Operations on Helper Table part -static void tsdbResetHelperTableImpl(SRWHelper *pHelper) { - memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable)); - pHelper->hasOldLastBlock = false; -} - -static void tsdbResetHelperTable(SRWHelper *pHelper) { - tsdbResetHelperBlock(pHelper); - tsdbResetHelperTableImpl(pHelper); - helperClearState(pHelper, (TSDB_HELPER_TABLE_SET | TSDB_HELPER_INFO_LOAD)); -} - -static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); } - -static void tsdbDestroyHelperTable(SRWHelper *pHelper) { taosTZfree((void *)pHelper->pCompInfo); } - -// ---------- Operations on Helper Block part -static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { - tdResetDataCols(pHelper->pDataCols[0]); - tdResetDataCols(pHelper->pDataCols[1]); -} - -static void tsdbResetHelperBlock(SRWHelper *pHelper) { - tsdbResetHelperBlockImpl(pHelper); - // helperClearState(pHelper, TSDB_HELPER_) -} - -static int tsdbInitHelperBlock(SRWHelper *pHelper) { - STsdbRepo *pRepo = helperRepo(pHelper); - STsdbMeta *pMeta = pHelper->pRepo->tsdbMeta; - - pHelper->pDataCols[0] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); - pHelper->pDataCols[1] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); - if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - tsdbResetHelperBlockImpl(pHelper); - - return 0; -} - -static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { - taosTZfree(pHelper->pCompData); - tdFreeDataCols(pHelper->pDataCols[0]); - tdFreeDataCols(pHelper->pDataCols[1]); -} - -static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) { - STsdbCfg *pCfg = &pRepo->config; - memset((void *)pHelper, 0, sizeof(*pHelper)); - STsdbMeta *pMeta = pRepo->tsdbMeta; - - helperType(pHelper) = type; - helperRepo(pHelper) = pRepo; - helperState(pHelper) = TSDB_HELPER_CLEAR_STATE; - - // Init file part - if (tsdbInitHelperFile(pHelper) < 0) goto _err; - - // Init table part - tsdbInitHelperTable(pHelper); - - // Init block part - if (tsdbInitHelperBlock(pHelper) < 0) goto _err; - - // TODO: pMeta->maxRowBytes and pMeta->maxCols may change here causing invalid write - pHelper->pBuffer = - taosTMalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pMeta->maxCols + - pMeta->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM)); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - return 0; - -_err: - tsdbDestroyHelper(pHelper); - return -1; -} - -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, - int maxPoints, char *buffer, int bufferSize) { - // Verify by checksum - if (!taosCheckChecksumWhole((uint8_t *)content, len)) { - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - - // Decode the data - if (comp) { - // // Need to decompress - int tlen = (*(tDataTypeDesc[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, - pDataCol->spaceSize, comp, buffer, bufferSize); - if (tlen <= 0) { - tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", - len, comp, numOfRows, maxPoints, bufferSize); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - return -1; - } - pDataCol->len = tlen; - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { - dataColSetOffset(pDataCol, numOfRows); - } - } else { - // No need to decompress, just memcpy it - pDataCol->len = len - sizeof(TSCKSUM); - memcpy(pDataCol->pData, content, pDataCol->len); - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { - dataColSetOffset(pDataCol, numOfRows); - } - } - return 0; -} - -static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBlock, SCompCol *pCompCol, - SDataCol *pDataCol) { - ASSERT(pDataCol->colId == pCompCol->colId); - int tsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; - pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pCompCol->len); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, tsize); - if (pHelper->compBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - int64_t offset = pCompBlock->offset + TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols) + pCompCol->offset; - if (lseek(pFile->fd, (off_t)offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) { - tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (tsdbCheckAndDecodeColumnData(pDataCol, pHelper->pBuffer, pCompCol->len, pCompBlock->algorithm, - pCompBlock->numOfRows, pHelper->pRepo->config.maxRowsPerFileBlock, - pHelper->compBuffer, (int32_t)taosTSizeof(pHelper->compBuffer)) < 0) { - tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, - pCompCol->colId, offset); - return -1; - } - - return 0; -} - -static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds, int numOfColIds) { - ASSERT(pCompBlock->numOfSubBlocks <= 1); - ASSERT(colIds[0] == 0); - - SFile * pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); - SCompCol compCol = {0}; - - // If only load timestamp column, no need to load SCompData part - if (numOfColIds > 1 && tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) goto _err; - - pDataCols->numOfRows = pCompBlock->numOfRows; - - int dcol = 0; - int ccol = 0; - for (int i = 0; i < numOfColIds; i++) { - int16_t colId = colIds[i]; - SDataCol *pDataCol = NULL; - SCompCol *pCompCol = NULL; - - while (true) { - if (dcol >= pDataCols->numOfCols) { - pDataCol = NULL; - break; - } - pDataCol = &pDataCols->cols[dcol]; - if (pDataCol->colId > colId) { - pDataCol = NULL; - break; - } else { - dcol++; - if (pDataCol->colId == colId) break; - } - } - - if (pDataCol == NULL) continue; - ASSERT(pDataCol->colId == colId); - - if (colId == 0) { // load the key row - compCol.colId = colId; - compCol.len = pCompBlock->keyLen; - compCol.type = pDataCol->type; - compCol.offset = TSDB_KEY_COL_OFFSET; - pCompCol = &compCol; - } else { // load non-key rows - while (true) { - if (ccol >= pCompBlock->numOfCols) { - pCompCol = NULL; - break; - } - - pCompCol = &(pHelper->pCompData->cols[ccol]); - if (pCompCol->colId > colId) { - pCompCol = NULL; - break; - } else { - ccol++; - if (pCompCol->colId == colId) break; - } - } - - if (pCompCol == NULL) { - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - continue; - } - - ASSERT(pCompCol->colId == pDataCol->colId); - } - - if (tsdbLoadColData(pHelper, pFile, pCompBlock, pCompCol, pDataCol) < 0) goto _err; - } - - return 0; - -_err: - return -1; -} - -static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { - ASSERT(pCompBlock->numOfSubBlocks <= 1); - - SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); - - pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pCompBlock->len); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - - SCompData *pCompData = (SCompData *)pHelper->pBuffer; - - int fd = pFile->fd; - if (lseek(fd, (off_t)pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (taosRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) { - tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - int32_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols); - if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) { - tsdbError("vgId:%d file %s block data is corrupted offset %" PRId64 " len %d", REPO_ID(pHelper->pRepo), - pFile->fname, (int64_t)(pCompBlock->offset), pCompBlock->len); - terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - goto _err; - } - ASSERT(pCompData->numOfCols == pCompBlock->numOfCols); - - pDataCols->numOfRows = pCompBlock->numOfRows; - - // Recover the data - int ccol = 0; // loop iter for SCompCol object - int dcol = 0; // loop iter for SDataCols object - while (dcol < pDataCols->numOfCols) { - SDataCol *pDataCol = &(pDataCols->cols[dcol]); - if (dcol != 0 && ccol >= pCompData->numOfCols) { - // Set current column as NULL and forward - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - dcol++; - continue; - } - - int16_t tcolId = 0; - int32_t toffset = TSDB_KEY_COL_OFFSET; - int32_t tlen = pCompBlock->keyLen; - - if (dcol != 0) { - SCompCol *pCompCol = &(pCompData->cols[ccol]); - tcolId = pCompCol->colId; - toffset = pCompCol->offset; - tlen = pCompCol->len; - } else { - ASSERT(pDataCol->colId == tcolId); - } - - if (tcolId == pDataCol->colId) { - if (pCompBlock->algorithm == TWO_STAGE_COMP) { - int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { - zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows); - } - pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, zsize); - if (pHelper->compBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - goto _err; - } - } - if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + toffset, tlen, pCompBlock->algorithm, - pCompBlock->numOfRows, pDataCols->maxPoints, pHelper->compBuffer, - (int32_t)taosTSizeof(pHelper->compBuffer)) < 0) { - tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d", - REPO_ID(pHelper->pRepo), pFile->fname, tcolId, (int64_t)pCompBlock->offset, toffset); - goto _err; - } - if (dcol != 0) ccol++; - dcol++; - } else if (tcolId < pDataCol->colId) { - ccol++; - } else { - // Set current column as NULL and forward - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - dcol++; - } - } - - return 0; - -_err: - return -1; -} - -static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) { - int tlen = 0; - - tlen += taosEncodeVariantI32(buf, pIdx->tid); - tlen += taosEncodeVariantU32(buf, pIdx->len); - tlen += taosEncodeVariantU32(buf, pIdx->offset); - tlen += taosEncodeFixedU8(buf, pIdx->hasLast); - tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks); - tlen += taosEncodeFixedU64(buf, pIdx->uid); - tlen += taosEncodeFixedU64(buf, pIdx->maxKey); - - return tlen; -} - -static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { - uint8_t hasLast = 0; - uint32_t numOfBlocks = 0; - uint64_t value = 0; - - if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL; - if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; - if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; - if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; - pIdx->hasLast = hasLast; - if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; - pIdx->numOfBlocks = numOfBlocks; - if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->uid = (int64_t)value; - if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->maxKey = (TSKEY)value; - - return buf; -} - -static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SCompBlock compBlock = {0}; - SMergeInfo mergeInfo = {0}; - SMergeInfo *pMergeInfo = &mergeInfo; - - ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey); - if (pIdx->hasLast) { // append to with last block - ASSERT(pIdx->len > 0); - SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); - ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols, - NULL, 0, pCfg->update, pMergeInfo); - - ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - - if (pDataCols->numOfRows > 0) { - ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); - - if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && - pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; - pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst); - pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast); - if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1; - } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); - - if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows); - - if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; - } - - if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - } - } else { - ASSERT(!pHelper->hasOldLastBlock); - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); - ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - - if (pDataCols->numOfRows > 0) { - ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); - if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; - } - } - -#ifndef NDEBUG - TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter); - ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey); -#endif - - return 0; -} - -static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, - int *blkIdx) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - SCompBlock compBlock = {0}; - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SDataCols * pDataCols0 = pHelper->pDataCols[0]; - SMergeInfo mergeInfo = {0}; - SMergeInfo *pMergeInfo = &mergeInfo; - SCompBlock oBlock = {0}; - - SSkipListIterator slIter = {0}; - - ASSERT(keyFirst <= pIdx->maxKey); - - SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)blockAtIdx(pHelper, *blkIdx), - pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE); - ASSERT(pCompBlock != NULL); - int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); - oBlock = *pCompBlock; - - ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); - - if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { - while (true) { - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0, - pCfg->update, pMergeInfo); - ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - if (pDataCols->numOfRows == 0) break; - - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - tblkIdx++; - } - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); - } else { - int16_t colId = 0; - if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1; - - TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1); - - slIter = *(pCommitIter->pIter); - tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows, - pCfg->update, pMergeInfo); - - if (pMergeInfo->nOperations == 0) { - // Do nothing - ASSERT(pMergeInfo->rowsDeleteFailed >= 0); - *(pCommitIter->pIter) = slIter; - tblkIdx++; - } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { - // Delete the block and do some stuff - // ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN); - if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1; - *pCommitIter->pIter = slIter; - if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) { - // Append as a sub-block of the searched block - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData, - pDataCols0->numOfRows, pCfg->update, pMergeInfo); - ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0); - if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, - &compBlock, oBlock.last, false) < 0) { - return -1; - } - if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) { - return -1; - } - tblkIdx++; - } else { - // load the block data, merge with the memory data - if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1; - int round = 0; - int dIter = 0; - while (true) { - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock, - pCfg->update); - - if (pDataCols->numOfRows == 0) break; - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - - if (round == 0) { - if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } else { - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } - - round++; - tblkIdx++; - } - } - } - - *blkIdx = tblkIdx; - return 0; -} - -static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows, int8_t update) { - TSKEY key1 = INT64_MAX; - TSKEY key2 = INT64_MAX; - STSchema *pSchema = NULL; - - ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); - tdResetDataCols(pTarget); - - while (true) { - key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); - bool isRowDel = false; - SDataRow row = tsdbNextIterRow(pCommitIter->pIter); - if (row == NULL || dataRowKey(row) > maxKey) { - key2 = INT64_MAX; - } else { - key2 = dataRowKey(row); - isRowDel = dataRowDeleted(row); - } - - if (key1 == INT64_MAX && key2 == INT64_MAX) break; - - if (key1 < key2) { - for (int i = 0; i < pDataCols->numOfCols; i++) { - dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, - pTarget->maxPoints); - } - - pTarget->numOfRows++; - (*iter)++; - } else if (key1 > key2) { - if (!isRowDel) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); - ASSERT(pSchema != NULL); - } - - tdAppendDataRowToDataCol(row, pSchema, pTarget); - } - - tSkipListIterNext(pCommitIter->pIter); - } else { - if (update) { - if (!isRowDel) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); - ASSERT(pSchema != NULL); - } - - tdAppendDataRowToDataCol(row, pSchema, pTarget); - } - } else { - ASSERT(!isRowDel); - - for (int i = 0; i < pDataCols->numOfCols; i++) { - dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, - pTarget->maxPoints); - } - - pTarget->numOfRows++; - } - (*iter)++; - tSkipListIterNext(pCommitIter->pIter); - } - - if (pTarget->numOfRows >= maxRows) break; - } -} - -static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) { - STsdbCfg *pCfg = &(pHelper->pRepo->config); - SFile * pFile = NULL; - bool isLast = false; - - ASSERT(pDataCols->numOfRows > 0); - - if (pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { - pFile = helperDataF(pHelper); - } else { - isLast = true; - pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? helperNewLastF(pHelper) : helperLastF(pHelper); - } - - ASSERT(pFile->fd > 0); - - if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, pCompBlock, isLast, true) < 0) return -1; - - return 0; -} - -static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) { - STsdbCfg *pCfg = &(pHelper->pRepo->config); - int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; - - ASSERT(mergeRows > 0); - - if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) { - if (pCompBlock->last) { - if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true; - } else { - if (mergeRows < pCfg->maxRowsPerFileBlock) return true; - } - } - - return false; -} \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2444283435efeefdccdae2da5a047f23b7330ad5..ea72760568293043b0cb415ee1beb6c0866ab5f9 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -19,10 +19,9 @@ #include "tcompare.h" #include "exception.h" -#include "../../query/inc/qAst.h" // todo move to common module #include "tlosertree.h" -#include "tsdb.h" -#include "tsdbMain.h" +#include "tsdbint.h" +#include "texpr.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -35,11 +34,9 @@ .tid = (_checkInfo)->tableId.tid, \ .uid = (_checkInfo)->tableId.uid}) - enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, - TSDB_QUERY_TYPE_EXTERNAL = 3, }; typedef struct SQueryFilePos { @@ -54,7 +51,7 @@ typedef struct SQueryFilePos { } SQueryFilePos; typedef struct SDataBlockLoadInfo { - SFileGroup* fileGroup; + SDFileSet* fileGroup; int32_t slot; int32_t tid; SArray* pLoadedCols; @@ -69,7 +66,7 @@ typedef struct STableCheckInfo { STableId tableId; TSKEY lastKey; STable* pTableObj; - SCompInfo* pCompInfo; + SBlockInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks int8_t chosen:2; // indicate which iterator should move forward @@ -79,8 +76,8 @@ typedef struct STableCheckInfo { } STableCheckInfo; typedef struct STableBlockInfo { - SCompBlock* compBlock; - STableCheckInfo* pTableCheckInfo; + SBlock *compBlock; + STableCheckInfo *pTableCheckInfo; } STableBlockInfo; typedef struct SBlockOrderSupporter { @@ -111,11 +108,14 @@ typedef struct STsdbQueryHandle { int32_t activeIndex; bool checkFiles; // check file stage bool cachelastrow; // check if last row cached - void* qinfo; // query info handle, for debug purpose + bool loadExternalRow; // load time window external data rows + bool currentLoadExternalRows; // current load external rows + int32_t loadType; // block load type + uint64_t qId; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows - SFileGroup* pFileGroup; - SFileGroupIter fileIter; - SRWHelper rhelper; + SDFileSet* pFileGroup; + SFSIter fileIter; + SReadH rhelper; STableBlockInfo* pDataBlockInfo; SDataCols *pDataCols; // in order to hold current file data block @@ -125,6 +125,8 @@ typedef struct STsdbQueryHandle { SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */ + SArray *prev; // previous row which is before than time window + SArray *next; // next row which is after the query time window SIOCostSummary cost; } STsdbQueryHandle; @@ -139,12 +141,14 @@ static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroup static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle); -static int tsdbCheckInfoCompar(const void* key1, const void* key2); - +static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); +static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); +static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef); +static void* doFreeColumnInfoData(SArray* pColumnInfoData); +static void* destroyTableCheckInfo(SArray* pTableCheckInfo); +static bool tsdbGetExternalRow(TsdbQueryHandleT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -185,13 +189,15 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { +static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTable) { assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); SMemRef* pMemRef = pQueryHandle->pMemRef; if (pQueryHandle->pMemRef->ref++ == 0) { - tsdbTakeMemSnapshot(pQueryHandle->pTsdb, (SMemTable**)&(pMemRef->mem), (SMemTable**)&(pMemRef->imem)); + tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &(pMemRef->snapshot), psTable); } + + taosArrayDestroy(psTable); } static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { @@ -202,15 +208,43 @@ static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { } if (--pMemRef->ref == 0) { - tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem); - pMemRef->mem = NULL; - pMemRef->imem = NULL; + tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, &(pMemRef->snapshot)); } pQueryHandle->pMemRef = NULL; } -static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) { +int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(pQueryHandle->activeIndex < size && pQueryHandle->activeIndex >= 0 && size >= 1); + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + + + int64_t rows = 0; + SMemRef* pMemRef = pQueryHandle->pMemRef; + if (pMemRef == NULL) { return rows; } + + STableData* pMem = NULL; + STableData* pIMem = NULL; + + SMemTable* pMemT = pMemRef->snapshot.mem; + SMemTable* pIMemT = pMemRef->snapshot.imem; + + if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { + pMem = pMemT->tData[pCheckInfo->tableId.tid]; + rows += (pMem && pMem->uid == pCheckInfo->tableId.uid) ? pMem->numOfRows: 0; + } + if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { + pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; + rows += (pIMem && pIMem->uid == pCheckInfo->tableId.uid) ? pIMem->numOfRows: 0; + } + + return rows; +} + +static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta, SArray** psTable) { size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); assert(sizeOfGroup >= 1 && pMeta != NULL); @@ -220,6 +254,12 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa return NULL; } + SArray* pTable = taosArrayInit(4, sizeof(STable*)); + if (pTable == NULL) { + taosArrayDestroy(pTableCheckInfo); + return NULL; + } + // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(pGroupList->pGroupList, i); @@ -246,36 +286,60 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa } taosArrayPush(pTableCheckInfo, &info); - tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" %p", pQueryHandle, info.tableId.uid, - info.tableId.tid, info.lastKey, pQueryHandle->qinfo); + tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" %"PRIu64, pQueryHandle, info.tableId.uid, + info.tableId.tid, info.lastKey, pQueryHandle->qId); } } taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); + + size_t gsize = taosArrayGetSize(pTableCheckInfo); + + for (int32_t i = 0; i < gsize; ++i) { + STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); + taosArrayPush(pTable, &pInfo->pTableObj); + } + + *psTable = pTable; return pTableCheckInfo; } -static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey) { - size_t si = taosArrayGetSize(pTableCheckInfo); - SArray* pNew = taosArrayInit(si, sizeof(STableCheckInfo)); - if (pNew == NULL) { - return NULL; - } +static void resetCheckInfo(STsdbQueryHandle* pQueryHandle) { + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(numOfTables >= 1); - for (int32_t j = 0; j < si; ++j) { - STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, j); - STableCheckInfo info = { .lastKey = skey, .pTableObj = pCheckInfo->pTableObj}; + // todo apply the lastkey of table check to avoid to load header file + for (int32_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); + pCheckInfo->lastKey = pQueryHandle->window.skey; + pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter); + pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter); + pCheckInfo->initBuf = false; - info.tableId = pCheckInfo->tableId; - taosArrayPush(pNew, &info); + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + assert(pCheckInfo->lastKey >= pQueryHandle->window.skey); + } else { + assert(pCheckInfo->lastKey <= pQueryHandle->window.skey); + } } +} - // it is ordered already, no need to sort again. - taosArraySort(pNew, tsdbCheckInfoCompar); +// only one table, not need to sort again +static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY skey, SArray** psTable) { + SArray* pNew = taosArrayInit(1, sizeof(STableCheckInfo)); + SArray* pTable = taosArrayInit(1, sizeof(STable*)); + + STableCheckInfo info = { .lastKey = skey, .pTableObj = pCheckInfo->pTableObj}; + + info.tableId = pCheckInfo->tableId; + taosArrayPush(pNew, &info); + taosArrayPush(pTable, &pCheckInfo->pTableObj); + + *psTable = pTable; return pNew; } -static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) { +static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, uint64_t qId, SMemRef* pMemRef) { STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); if (pQueryHandle == NULL) { goto out_of_memory; @@ -285,23 +349,25 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; + pQueryHandle->cur.fid = INT32_MIN; pQueryHandle->cur.win = TSWINDOW_INITIALIZER; pQueryHandle->checkFiles = true; pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->qinfo = qinfo; + pQueryHandle->qId = qId; pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; pQueryHandle->allocSize = 0; pQueryHandle->locateStart = false; pQueryHandle->pMemRef = pMemRef; + pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows; - if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) { + pQueryHandle->loadType = pCond->type; + + if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) { goto out_of_memory; } - tsdbMayTakeMemSnapshot(pQueryHandle); assert(pCond != NULL && pCond->numOfCols > 0 && pMemRef != NULL); - if (ASCENDING_TRAVERSE(pCond->order)) { assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); } else { @@ -331,14 +397,16 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pQueryHandle->statis[i].colId = colInfo.info.colId; } - pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + if (pCond->numOfCols > 0) { + pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + } STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); if (pQueryHandle->pDataCols == NULL) { - tsdbError("%p failed to malloc buf for pDataCols, %p", pQueryHandle, pQueryHandle->qinfo); + tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto out_of_memory; } @@ -354,25 +422,97 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* return NULL; } -TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) { - STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef); +TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, SMemRef* pRef) { + STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); + SArray* psTable = NULL; + // todo apply the lastkey of table check to avoid to load header file - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta); + pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &psTable); if (pQueryHandle->pTableCheckInfo == NULL) { tsdbCleanupQueryHandle(pQueryHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } - tsdbDebug("%p total numOfTable:%" PRIzu " in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); + tsdbMayTakeMemSnapshot(pQueryHandle, psTable); + + tsdbDebug("%p total numOfTable:%" PRIzu " in query, %"PRIu64, pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qId); return (TsdbQueryHandleT) pQueryHandle; } -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) { +void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { + STsdbQueryHandle* pQueryHandle = queryHandle; + + pQueryHandle->order = pCond->order; + pQueryHandle->window = pCond->twindow; + pQueryHandle->type = TSDB_QUERY_TYPE_ALL; + pQueryHandle->cur.fid = -1; + pQueryHandle->cur.win = TSWINDOW_INITIALIZER; + pQueryHandle->checkFiles = true; + pQueryHandle->activeIndex = 0; // current active table index + pQueryHandle->locateStart = false; + pQueryHandle->loadExternalRow = pCond->loadExternalRows; + + if (ASCENDING_TRAVERSE(pCond->order)) { + assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } else { + assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + } + + // allocate buffer in order to load data blocks from file + memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + + tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + + resetCheckInfo(pQueryHandle); +} + +void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { + STsdbQueryHandle* pQueryHandle = queryHandle; + + pQueryHandle->order = pCond->order; + pQueryHandle->window = pCond->twindow; + pQueryHandle->type = TSDB_QUERY_TYPE_ALL; + pQueryHandle->cur.fid = -1; + pQueryHandle->cur.win = TSWINDOW_INITIALIZER; + pQueryHandle->checkFiles = true; + pQueryHandle->activeIndex = 0; // current active table index + pQueryHandle->locateStart = false; + pQueryHandle->loadExternalRow = pCond->loadExternalRows; + + if (ASCENDING_TRAVERSE(pCond->order)) { + assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } else { + assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + } + + // allocate buffer in order to load data blocks from file + memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + + tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + + SArray* pTable = NULL; + STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); + + pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); + + pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &pTable); + if (pQueryHandle->pTableCheckInfo == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + } + + pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); + pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); +} + +TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -380,7 +520,7 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab return NULL; } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pMemRef); + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); int32_t code = checkForCachedLastRow(pQueryHandle, groupList); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; @@ -408,12 +548,46 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { return res; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef); - if (pQueryHandle != NULL) { - pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; - changeQueryHandleForInterpQuery(pQueryHandle); +// leave only one table for each group +static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGroupList) { + assert(pGroupList); + size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); + + STableGroupInfo* pNew = calloc(1, sizeof(STableGroupInfo)); + pNew->pGroupList = taosArrayInit(numOfGroup, sizeof(SArray)); + + for(int32_t i = 0; i < numOfGroup; ++i) { + SArray* oneGroup = taosArrayGetP(pGroupList->pGroupList, i); + size_t numOfTables = taosArrayGetSize(oneGroup); + + SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); + for (int32_t j = 0; j < numOfTables; ++j) { + STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); + if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { + taosArrayPush(px, pInfo); + pNew->numOfTables += 1; + break; + } + } + + // there are no data in this group + if (taosArrayGetSize(px) == 0) { + taosArrayDestroy(px); + } else { + taosArrayPush(pNew->pGroupList, &px); + } } + + return pNew; +} + +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pRef) { + STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); + + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); + pQueryHandle->loadExternalRow = true; + pQueryHandle->currentLoadExternalRows = true; + return pQueryHandle; } @@ -429,7 +603,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh int32_t order = pHandle->order; // no data in buffer, abort - if (pHandle->pMemRef->mem == NULL && pHandle->pMemRef->imem == NULL) { + if (pHandle->pMemRef->snapshot.mem == NULL && pHandle->pMemRef->snapshot.imem == NULL) { return false; } @@ -438,22 +612,24 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh STableData* pMem = NULL; STableData* pIMem = NULL; - SMemTable* pMemT = pHandle->pMemRef->mem; - SMemTable* pIMemT = pHandle->pMemRef->imem; + SMemTable* pMemT = pHandle->pMemRef->snapshot.mem; + SMemTable* pIMemT = pHandle->pMemRef->snapshot.imem; if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { pMem = pMemT->tData[pCheckInfo->tableId.tid]; if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid + TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); pCheckInfo->iter = - tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal(pMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid + TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); pCheckInfo->iiter = - tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal(pIMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } @@ -475,9 +651,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", + "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %"PRIu64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, - pCheckInfo->lastKey, pMem->numOfRows, pHandle->qinfo); + pCheckInfo->lastKey, pMem->numOfRows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -486,8 +662,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, - pHandle->qinfo); + tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %"PRIu64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + pHandle->qId); } if (!imemEmpty) { @@ -497,9 +673,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", + "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %"PRIu64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, - pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qinfo); + pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -507,8 +683,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->lastKey >= key); } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, - pHandle->qinfo); + tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %"PRIu64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + pHandle->qId); } return true; @@ -618,7 +794,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); - pHandle->cur.fid = -1; + pHandle->cur.fid = INT32_MIN; STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); @@ -635,8 +811,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { } pCheckInfo->lastKey = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo); + tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %"PRIu64, pHandle, + pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId); // all data in mem are checked already. if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) || @@ -666,6 +842,10 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio return INT32_MIN; } + if (key < 0) { + key -= (daysPerFile * tsMsPerDay[precision]); + } + int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision])); // set the starting fileId if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32 fid = INT32_MIN; @@ -678,7 +858,7 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio return (int32_t)fid; } -static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { +static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { int32_t firstSlot = 0; int32_t lastSlot = numOfBlocks - 1; @@ -705,103 +885,122 @@ static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSK return midSlot; } -static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { - // load all the comp offset value for all tables in this file - int32_t code = TSDB_CODE_SUCCESS; +static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { + int32_t code = 0; - *numOfBlocks = 0; - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); + pCheckInfo->numOfBlocks = 0; - for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - pCheckInfo->numOfBlocks = 0; + if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { + code = terrno; + return code; + } - if (tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb) != TSDB_CODE_SUCCESS) { - code = terrno; - break; - } + SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; - SCompIdx* compIndex = &pQueryHandle->rhelper.curCompIdx; + // no data block in this file, try next file + if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { + return 0; // no data blocks in the file belongs to pCheckInfo->pTable + } - // no data block in this file, try next file - if (compIndex->len == 0 || compIndex->numOfBlocks == 0 || compIndex->uid != pCheckInfo->tableId.uid) { - continue; // no data blocks in the file belongs to pCheckInfo->pTable + if (pCheckInfo->compSize < (int32_t)compIndex->len) { + assert(compIndex->len > 0); + + char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); + if (t == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_TDB_OUT_OF_MEMORY; + return code; } - if (pCheckInfo->compSize < (int32_t)compIndex->len) { - assert(compIndex->len > 0); + pCheckInfo->pCompInfo = (SBlockInfo*)t; + pCheckInfo->compSize = compIndex->len; + } - char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); - if (t == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - code = TSDB_CODE_TDB_OUT_OF_MEMORY; - break; - } + if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { + return terrno; + } + SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; - pCheckInfo->pCompInfo = (SCompInfo*) t; - pCheckInfo->compSize = compIndex->len; - } + TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; - tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); - SCompInfo* pCompInfo = pCheckInfo->pCompInfo; + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } else { + assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); + } - TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; + s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); + e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); - } else { - assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); - } + // discard the unqualified data block based on the query time window + int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); + int32_t end = start; - s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); - e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); + if (s > pCompInfo->blocks[start].keyLast) { + return 0; + } - // discard the unqualified data block based on the query time window - int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); - int32_t end = start; + // todo speedup the procedure of located end block + while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + end += 1; + } - if (s > pCompInfo->blocks[start].keyLast) { - continue; - } + pCheckInfo->numOfBlocks = (end - start); - // todo speedup the procedure of located end block - while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { - end += 1; - } + if (start > 0) { + memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock)); + } - pCheckInfo->numOfBlocks = (end - start); + (*numOfBlocks) += pCheckInfo->numOfBlocks; + return 0; +} - if (start > 0) { - memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SCompBlock)); - } +static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { + // load all the comp offset value for all tables in this file + int32_t code = TSDB_CODE_SUCCESS; + *numOfBlocks = 0; + + size_t numOfTables = 0; + if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); + } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { + numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - (*numOfBlocks) += pCheckInfo->numOfBlocks; + for (int32_t i = 0; i < numOfTables; ++i) { + code = loadBlockInfo(pQueryHandle, i, numOfBlocks); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } else { + assert(0); } return code; } -static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { +static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { int64_t st = taosGetTimestampUs(); STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); int32_t code = tdInitDataCols(pQueryHandle->pDataCols, pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for pDataCols, %p", pQueryHandle, pQueryHandle->qinfo); + tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema); + code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %p", pQueryHandle, pQueryHandle->qinfo); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %"PRIu64, pQueryHandle, pQueryHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema); + code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %p", pQueryHandle, pQueryHandle->qinfo); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %"PRIu64, pQueryHandle, pQueryHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } @@ -821,23 +1020,31 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p pBlockLoadInfo->slot = pQueryHandle->cur.slot; pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid; - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); pBlock->numOfRows = pCols->numOfRows; + // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z + if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + int64_t* src = pCols->cols[0].pData; + for(int32_t i = 0; i < pBlock->numOfRows; ++i) { + src[i] = tdGetKey(src[i]); + } + } + int64_t elapsedTime = (taosGetTimestampUs() - st); pQueryHandle->cost.blockLoadTime += elapsedTime; - tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, %p", - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qinfo); + tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, %"PRIu64, + pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qId); return TSDB_CODE_SUCCESS; _error: pBlock->numOfRows = 0; - tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, %p", - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pQueryHandle->qinfo); + tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, %"PRIu64, + pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pQueryHandle->qId); return terrno; } @@ -847,7 +1054,7 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); -static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){ +static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ SQueryFilePos* cur = &pQueryHandle->cur; STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); @@ -859,7 +1066,7 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc assert(cur->pos >= 0 && cur->pos <= binfo.rows); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; - tsdbDebug("%p key in mem:%"PRId64", %p", pQueryHandle, key, pQueryHandle->qinfo); + tsdbDebug("%p key in mem:%"PRId64", %"PRIu64, pQueryHandle, key, pQueryHandle->qId); if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { @@ -930,7 +1137,7 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc return code; } -static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { +static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { SQueryFilePos* cur = &pQueryHandle->cur; int32_t code = TSDB_CODE_SUCCESS; @@ -942,7 +1149,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); if (pCheckInfo->lastKey > pBlock->keyFirst) { @@ -965,7 +1172,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl return code; } - SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); } else { @@ -1050,7 +1257,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity char* pData = NULL; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; int32_t num = end - start + 1; @@ -1082,7 +1289,6 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity } if (pColInfo->info.colId == src->colId) { - if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) { memmove(pData, (char*)src->pData + bytes * start, bytes * num); } else { // handle the var-string @@ -1142,13 +1348,18 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity } static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row, - int32_t numOfCols, STable* pTable) { + int32_t numOfCols, STable* pTable, STSchema* pSchema) { char* pData = NULL; // the schema version info is embeded in SDataRow - STSchema* pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); - int32_t numOfRowCols = schemaNCols(pSchema); - + int32_t numOfRowCols = 0; + if (pSchema == NULL) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + numOfRowCols = schemaNCols(pSchema); + } else { + numOfRowCols = schemaNCols(pSchema); + } + int32_t i = 0, j = 0; while(i < numOfCols && j < numOfRowCols) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); @@ -1165,10 +1376,40 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if (pSchema->columns[j].colId == pColInfo->info.colId) { void* value = tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - memcpy(pData, value, varDataTLen(value)); - } else { - memcpy(pData, value, pColInfo->info.bytes); + switch (pColInfo->info.type) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + memcpy(pData, value, varDataTLen(value)); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *(uint8_t *)pData = *(uint8_t *)value; + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *(uint16_t *)pData = *(uint16_t *)value; + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *(uint32_t *)pData = *(uint32_t *)value; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t *)pData = *(uint64_t *)value; + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_PTR(pData, value); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_PTR(pData, value); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + *(TSKEY *)pData = tdGetKey(*(TKEY *)value); + break; + default: + memcpy(pData, value, pColInfo->info.bytes); } j++; @@ -1274,7 +1515,7 @@ static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) { static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { SQueryFilePos* cur = &pQueryHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; @@ -1306,9 +1547,9 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %"PRIu64, pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qinfo); + cur->win.ekey, cur->rows, pQueryHandle->qId); } int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { @@ -1317,7 +1558,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; SQueryFilePos* cur = &pQueryHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { endPos = pBlockInfo->rows - 1; @@ -1336,14 +1577,14 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; initTableMemIterator(pQueryHandle, pCheckInfo); - SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0]; + SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && cur->pos >= 0 && cur->pos < pBlock->numOfRows); @@ -1360,13 +1601,16 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," - "end:%d, %p", + "end:%d, %"PRIu64, pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, - blockInfo.rows, cur->pos, endPos, pQueryHandle->qinfo); + blockInfo.rows, cur->pos, endPos, pQueryHandle->qId); // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; + int16_t rv = -1; + STSchema* pSchema = NULL; + int32_t pos = cur->pos; cur->win = TSWINDOW_INITIALIZER; @@ -1388,14 +1632,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* break; } - if (((tsArray[pos] > pQueryHandle->window.ekey || pos > endPos) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((tsArray[pos] < pQueryHandle->window.ekey || pos < endPos) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if (((pos > endPos || tsArray[pos] > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((pos < endPos || tsArray[pos] < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { break; } if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1408,7 +1657,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1489,9 +1743,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %"PRIu64, pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qinfo); + cur->win.ekey, cur->rows, pQueryHandle->qId); } int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { @@ -1613,6 +1867,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO memset(pQueryHandle->pDataBlockInfo, 0, size); *numOfAllocBlocks = numOfBlocks; + // access data blocks according to the offset of each block in asc/desc order. int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); SBlockOrderSupporter sup = {0}; @@ -1635,7 +1890,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO continue; } - SCompBlock* pBlock = pTableCheck->pCompInfo->blocks; + SBlock* pBlock = pTableCheck->pCompInfo->blocks; sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks; char* buf = calloc(1, sizeof(STableBlockInfo) * pTableCheck->numOfBlocks); @@ -1664,13 +1919,13 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO memcpy(pQueryHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); cleanBlockOrderSupporter(&sup, numOfQualTables); - tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted %p ", pQueryHandle, cnt, - pQueryHandle->qinfo); + tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted %"PRIu64, pQueryHandle, cnt, + pQueryHandle->qId); return TSDB_CODE_SUCCESS; } - tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %p", pQueryHandle, cnt, - numOfQualTables, pQueryHandle->qinfo); + tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %"PRIu64, pQueryHandle, cnt, + numOfQualTables, pQueryHandle->qId); assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; @@ -1706,7 +1961,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - tsdbDebug("%p %d data blocks sort completed, %p", pQueryHandle, cnt, pQueryHandle->qinfo); + tsdbDebug("%p %d data blocks sort completed, %"PRIu64, pQueryHandle, cnt, pQueryHandle->qId); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); @@ -1751,35 +2006,35 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist STimeWindow win = TSWINDOW_INITIALIZER; while (true) { - pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) == NULL) { - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fileId, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo); + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %"PRIu64, pQueryHandle, + pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); pQueryHandle->pFileGroup = NULL; assert(pQueryHandle->numOfBlocks == 0); break; } - if (tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); code = terrno; break; } - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - if (tsdbLoadCompIdx(&pQueryHandle->rhelper, NULL) < 0) { + if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { code = terrno; break; } @@ -1788,8 +2043,8 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %"PRIu64, pQueryHandle, numOfBlocks, numOfTables, + pQueryHandle->pFileGroup->fid, pQueryHandle->qId); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -1813,14 +2068,14 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist assert(pQueryHandle->pFileGroup == NULL); } - cur->fid = -1; // denote that there are no data in file anymore + cur->fid = INT32_MIN; // denote that there are no data in file anymore *exists = false; return code; } assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fileId; + cur->fid = pQueryHandle->pFileGroup->fid; STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; return getDataBlockRv(pQueryHandle, pBlockInfo, exists); @@ -1842,8 +2097,93 @@ static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { cur->blockCompleted = false; } +int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; + + pTableBlockInfo->totalSize = 0; + STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); + + // find the start data block in file + pQueryHandle->locateStart = true; + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + + tsdbRLockFS(pFileHandle); + tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); + tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbUnLockFS(pFileHandle); + + pTableBlockInfo->numOfFiles += 1; + + int32_t code = TSDB_CODE_SUCCESS; + int32_t numOfBlocks = 0; + int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + STimeWindow win = TSWINDOW_INITIALIZER; + + while (true) { + numOfBlocks = 0; + tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + + if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + break; + } + + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + + // current file are not overlapped with query time window, ignore remain files + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %"PRIu64, pQueryHandle, + pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); + pQueryHandle->pFileGroup = NULL; + break; + } + + pTableBlockInfo->numOfFiles += 1; + if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + code = terrno; + break; + } + + tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + + if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + code = terrno; + break; + } + + if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + break; + } + + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %"PRIu64, pQueryHandle, numOfBlocks, numOfTables, + pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + + if (numOfBlocks == 0) { + continue; + } + + for (int32_t i = 0; i < numOfTables; ++i) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + + SBlock* pBlock = pCheckInfo->pCompInfo->blocks; + for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) { + pTableBlockInfo->totalSize += pBlock[j].len; + + int32_t numOfRows = pBlock[j].numOfRows; + taosArrayPush(pTableBlockInfo->dataBlockInfos, &numOfRows); + } + } + } + + return code; +} + static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { - STsdbFileH* pFileHandle = tsdbGetFile(pQueryHandle->pTsdb); + STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); SQueryFilePos* cur = &pQueryHandle->cur; // find the start data block in file @@ -1852,10 +2192,10 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); - pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); - tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); - tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); - pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + tsdbRLockFS(pFileHandle); + tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); + tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbUnLockFS(pFileHandle); return getFirstFileDataBlock(pQueryHandle, exists); } else { @@ -1867,8 +2207,8 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists if ((!cur->mixBlock) || cur->blockCompleted) { // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, - pQueryHandle->qinfo); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, %"PRIu64, pQueryHandle, cur->slot, cur->pos, + pQueryHandle->qId); int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); *exists = (pQueryHandle->realNumOfRows > 0); @@ -1901,16 +2241,15 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { } // no data in memtable or imemtable, decrease the memory reference. - tsdbMayUnTakeMemSnapshot(pQueryHandle); + // TODO !! +// tsdbMayUnTakeMemSnapshot(pQueryHandle); return false; } -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { +//todo not unref yet, since it is not support multi-group interpolation query +static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { // filter the queried time stamp in the first place STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - pQueryHandle->order = TSDB_ORDER_DESC; - - assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); // starts from the buffer in case of descending timestamp order check data blocks size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); @@ -1920,8 +2259,8 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); // the first qualified table for interpolation query - if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && - pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { + if ((pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && + (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { break; } @@ -1938,9 +2277,6 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { info.lastKey = pQueryHandle->window.skey; taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; } static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, @@ -1952,6 +2288,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int int64_t st = taosGetTimestampUs(); STable* pTable = pCheckInfo->pTableObj; + int16_t rv = -1; + STSchema* pSchema = NULL; do { SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); @@ -1972,7 +2310,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); + if (rv != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); + rv = dataRowVersion(row); + } + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable, pSchema); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); @@ -1994,8 +2336,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %"PRIu64, pQueryHandle, + elapsedTime, numOfRows, numOfCols, pQueryHandle->qId); return numOfRows; } @@ -2020,7 +2362,6 @@ static void destroyHelper(void* param) { return; } - tQueryInfo* pInfo = (tQueryInfo*)param; if (pInfo->optr != TSDB_RELATION_IN) { tfree(pInfo->q); @@ -2029,193 +2370,304 @@ static void destroyHelper(void* param) { free(param); } -static bool getNeighborRows(STsdbQueryHandle* pQueryHandle) { - assert (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL); +static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { + if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block + bool exists = true; + + int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + if (code != TSDB_CODE_SUCCESS) { + pQueryHandle->checkFiles = false; + return false; + } - SDataBlockInfo blockInfo = {{0}, 0}; + if (exists) { + if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, 0); + assert(*(int64_t*)pColInfo->pData == pQueryHandle->window.skey); + } - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->order = TSDB_ORDER_DESC; + pQueryHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. + return exists; + } - if (!tsdbNextDataBlock((void*) pQueryHandle)) { - return false; + pQueryHandle->checkFiles = false; } - tsdbRetrieveDataBlockInfo((void*) pQueryHandle, &blockInfo); - /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pQueryHandle, pQueryHandle->defaultLoadColumn); - if (terrno != TSDB_CODE_SUCCESS) { - return false; + if (hasMoreDataInCache(pQueryHandle)) { + pQueryHandle->currentLoadExternalRows = false; + return true; } - if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) { - // data already retrieve, discard other data rows and return - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes); - } + // current result is empty + if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey && pQueryHandle->cur.rows == 0) { + SMemRef* pMemRef = pQueryHandle->pMemRef; - pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey}; - pQueryHandle->window = pQueryHandle->cur.win; - pQueryHandle->cur.rows = 1; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - return true; - } else { - STimeWindow win = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX}; - STsdbQueryCond cond = { - .order = TSDB_ORDER_ASC, - .numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)) - }; - cond.twindow = win; - - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - if (cond.colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return false; - } + doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); + doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); - for(int32_t i = 0; i < cond.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); - } + bool result = tsdbGetExternalRow(pQueryHandle); - STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef); + pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); + pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + pQueryHandle->currentLoadExternalRows = false; - tfree(cond.colList); + return result; + } - pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey); - if (pSecQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pSecQueryHandle); - return false; - } + return false; +} + +static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { + // the last row is cached in buffer, return it directly. + // here note that the pQueryHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(numOfTables > 0 && numOfCols > 0); - if (!tsdbNextDataBlock((void*) pSecQueryHandle)) { - tsdbCleanupQueryHandle(pSecQueryHandle); + SQueryFilePos* cur = &pQueryHandle->cur; + + SDataRow pRow = NULL; + TSKEY key = TSKEY_INITIAL_VAL; + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + + if (++pQueryHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); + if (ret != TSDB_CODE_SUCCESS) { return false; } - tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo); - tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); + tfree(pRow); - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pSecQueryHandle)); - size_t si = taosArrayGetSize(pSecQueryHandle->pTableCheckInfo); + // update the last key value + pCheckInfo->lastKey = key + step; - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes); + cur->rows = 1; // only one row + cur->lastKey = key + step; + cur->mixBlock = true; + cur->win.skey = key; + cur->win.ekey = key; - SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i); - assert(pCol->info.colId == pCol1->info.colId); + return true; + } - memcpy((char*)pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes); - } + return false; +} - SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0); +static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + assert(numOfTables > 0); - // it is ascending order - pQueryHandle->order = TSDB_ORDER_DESC; - pQueryHandle->window = pQueryHandle->cur.win; - pQueryHandle->cur.win = (STimeWindow){((TSKEY*)pTSCol->pData)[0], ((TSKEY*)pTSCol->pData)[1]}; - pQueryHandle->cur.rows = 2; - pQueryHandle->cur.mixBlock = true; + int64_t stime = taosGetTimestampUs(); - int32_t step = -1;// one step for ascending order traverse - for (int32_t j = 0; j < si; ++j) { - STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j); - pCheckInfo->lastKey = pQueryHandle->cur.win.ekey + step; + while(pQueryHandle->activeIndex < numOfTables) { + if (loadBlockOfActiveTable(pQueryHandle)) { + return true; } - tsdbCleanupQueryHandle(pSecQueryHandle); + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + pCheckInfo->numOfBlocks = 0; + + pQueryHandle->activeIndex += 1; + pQueryHandle->locateStart = false; + pQueryHandle->checkFiles = true; + pQueryHandle->cur.rows = 0; + pQueryHandle->currentLoadExternalRows = pQueryHandle->loadExternalRow; + + terrno = TSDB_CODE_SUCCESS; + + int64_t elapsedTime = taosGetTimestampUs() - stime; + pQueryHandle->cost.checkForNextTime += elapsedTime; } - //disable it after retrieve data - pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; - pQueryHandle->checkFiles = false; - return true; + return false; } // handle data in cache situation -bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { +bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; int64_t stime = taosGetTimestampUs(); int64_t elapsedTime = stime; - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - assert(numOfTables > 0); + if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { + return loadCachedLastRow(pQueryHandle); + } - if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) { - SMemRef* pMemRef = pQueryHandle->pMemRef; - tsdbMayTakeMemSnapshot(pQueryHandle); - bool ret = getNeighborRows(pQueryHandle); - tsdbMayUnTakeMemSnapshot(pQueryHandle); + if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + return loadDataBlockFromTableSeq(pQueryHandle); + } else { // loadType == RR and Offset Order + if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block + bool exists = true; + + int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + if (code != TSDB_CODE_SUCCESS) { + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; - // restore the pMemRef - pQueryHandle->pMemRef = pMemRef; - return ret; - } else if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST && pQueryHandle->cachelastrow) { - // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - SQueryFilePos* cur = &pQueryHandle->cur; - - SDataRow pRow = NULL; - TSKEY key = TSKEY_INITIAL_VAL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - - if (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); - if (ret != TSDB_CODE_SUCCESS) { return false; } - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj); - tfree(pRow); + if (exists) { + pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + return exists; + } - // update the last key value - pCheckInfo->lastKey = key + step; + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; + } - cur->rows = 1; // only one row - cur->lastKey = key + step; - cur->mixBlock = true; - cur->win.skey = key; - cur->win.ekey = key; + // TODO: opt by consider the scan order + bool ret = doHasDataInBuffer(pQueryHandle); + terrno = TSDB_CODE_SUCCESS; - return true; - } + elapsedTime = taosGetTimestampUs() - stime; + pQueryHandle->cost.checkForNextTime += elapsedTime; + return ret; + } +} - return false; +static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) { + STsdbQueryHandle* pSecQueryHandle = NULL; + + if (type == TSDB_PREV_ROW && pQueryHandle->prev) { + return TSDB_CODE_SUCCESS; } - if (pQueryHandle->checkFiles) { - // check if the query range overlaps with the file data block - bool exists = true; + if (type == TSDB_NEXT_ROW && pQueryHandle->next) { + return TSDB_CODE_SUCCESS; + } - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); - if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + // prepare the structure + int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); - return false; + if (type == TSDB_PREV_ROW) { + pQueryHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pQueryHandle->prev == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; } + } else { + pQueryHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pQueryHandle->next == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; + } + } - if (exists) { - pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); - return exists; + SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + + SColumnInfoData colInfo = {{0}, 0}; + colInfo.info = pCol->info; + colInfo.pData = calloc(1, pCol->info.bytes); + if (colInfo.pData == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; } - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + taosArrayPush(row, &colInfo); } - // TODO: opt by consider the scan order - bool ret = doHasDataInBuffer(pQueryHandle); - terrno = TSDB_CODE_SUCCESS; + // load the previous row + STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; + if (type == TSDB_PREV_ROW) { + cond.order = TSDB_ORDER_DESC; + cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN}; + } else { + cond.order = TSDB_ORDER_ASC; + cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MAX}; + } - elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; - return ret; + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + if (cond.colList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; + } + + for (int32_t i = 0; i < cond.numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); + memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); + } + + pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qId, pMemRef); + tfree(cond.colList); + + // current table, only one table + STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + + SArray* psTable = NULL; + pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); + if (pSecQueryHandle->pTableCheckInfo == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto out_of_memory; + } + + + tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); + if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { + // no result in current query, free the corresponding result rows structure + if (type == TSDB_PREV_ROW) { + pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); + } else { + pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + } + + goto out_of_memory; + } + + SDataBlockInfo blockInfo = {{0}, 0}; + tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); + tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); + + row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; + int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(row, i); + SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); + memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); + } + +out_of_memory: + tsdbCleanupQueryHandle(pSecQueryHandle); + return terrno; +} + +bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + SQueryFilePos* cur = &pQueryHandle->cur; + + cur->fid = INT32_MIN; + cur->mixBlock = true; + if (pQueryHandle->prev == NULL || pQueryHandle->next == NULL) { + cur->rows = 0; + return false; + } + + int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* first = taosArrayGet(pQueryHandle->prev, i); + + memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); + + SColumnInfoData* sec = taosArrayGet(pQueryHandle->next, i); + memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes); + + if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + cur->win.skey = *(TSKEY*)pColInfoData->pData; + cur->win.ekey = *(TSKEY*)(((char*)pColInfoData->pData) + TSDB_KEYSIZE); + } + } + + cur->rows = 2; + return true; } /* @@ -2345,7 +2797,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p STable* pTable = NULL; // there are data in file - if (pHandle->cur.fid >= 0) { + if (pHandle->cur.fid != INT32_MIN) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; pTable = pBlockInfo->pTableCheckInfo->pTableObj; } else { @@ -2382,7 +2834,9 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta } int64_t stime = taosGetTimestampUs(); - tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL); + if (tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock) < 0) { + return terrno; + } int16_t* colIds = pHandle->defaultLoadColumn->pData; @@ -2392,7 +2846,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta pHandle->statis[i].colId = colIds[i]; } - tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols); + tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols); // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; @@ -2429,7 +2883,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { */ STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; - if (pHandle->cur.fid < 0) { + if (pHandle->cur.fid == INT32_MIN) { return pHandle->pColumns; } else { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; @@ -2444,11 +2898,11 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; - if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid && + if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { return pHandle->pColumns; } else { // only load the file block - SCompBlock* pBlock = pBlockInfo->compBlock; + SBlock* pBlock = pBlockInfo->compBlock; if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) { return NULL; } @@ -2524,7 +2978,7 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa f1 = (char*) TABLE_NAME(pTable1); f2 = (char*) TABLE_NAME(pTable2); type = TSDB_DATA_TYPE_BINARY; - bytes = tGetTableNameColumnSchema().bytes; + bytes = tGetTbnameColumnSchema()->bytes; } else { STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex); bytes = pCol->bytes; @@ -2716,11 +3170,11 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) }; getTableListfromSkipList(pExpr, pSTable->pIndex, pRes, &supp); - tExprTreeDestroy(&pExpr, destroyHelper); + tExprTreeDestroy(pExpr, destroyHelper); return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; @@ -2755,10 +3209,10 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co goto _error; } - pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%" PRIzu "", tsdb, pGroupInfo->numOfTables); + tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%u", tsdb, pGroupInfo->numOfTables); taosArrayDestroy(res); if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; @@ -2773,10 +3227,10 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co if (expr == NULL) { expr = exprTreeFromBinary(pTagCond, len); } else { - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL); tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); if (tagExpr != NULL) { - CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, tagExpr, NULL); + CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, tagExpr, NULL); tExprNode* tbnameExpr = expr; expr = calloc(1, sizeof(tExprNode)); if (expr == NULL) { @@ -2800,10 +3254,10 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co } END_TRY doQueryTableList(pTable, res, expr); - pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%" PRIzu ", belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, + tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); taosArrayDestroy(res); @@ -2815,7 +3269,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co return terrno; } -int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -2845,7 +3299,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, ST return terrno; } -int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; } @@ -2880,7 +3334,7 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa return terrno; } - pGroupInfo->numOfTables = taosArrayGetSize(group); + pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(group); if (pGroupInfo->numOfTables > 0) { taosArrayPush(pGroupInfo->pGroupList, &group); } else { @@ -2890,31 +3344,42 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa return TSDB_CODE_SUCCESS; } +static void* doFreeColumnInfoData(SArray* pColumnInfoData) { + if (pColumnInfoData == NULL) { + return NULL; + } + + size_t cols = taosArrayGetSize(pColumnInfoData); + for (int32_t i = 0; i < cols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pColumnInfoData, i); + tfree(pColInfo->pData); + } + + taosArrayDestroy(pColumnInfoData); + return NULL; +} + +static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { + size_t size = taosArrayGetSize(pTableCheckInfo); + for (int32_t i = 0; i < size; ++i) { + STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i); + destroyTableMemIterator(p); + + tfree(p->pCompInfo); + } + + taosArrayDestroy(pTableCheckInfo); + return NULL; +} + void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; if (pQueryHandle == NULL) { return; } - - if (pQueryHandle->pTableCheckInfo != NULL) { - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - destroyTableMemIterator(pTableCheckInfo); - - tfree(pTableCheckInfo->pCompInfo); - } - taosArrayDestroy(pQueryHandle->pTableCheckInfo); - } - if (pQueryHandle->pColumns != NULL) { - size_t cols = taosArrayGetSize(pQueryHandle->pColumns); - for (int32_t i = 0; i < cols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - tfree(pColInfo->pData); - } - taosArrayDestroy(pQueryHandle->pColumns); - } + pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); + pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); taosArrayDestroy(pQueryHandle->defaultLoadColumn); tfree(pQueryHandle->pDataBlockInfo); @@ -2923,14 +3388,17 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { // todo check error tsdbMayUnTakeMemSnapshot(pQueryHandle); - tsdbDestroyHelper(&pQueryHandle->rhelper); + tsdbDestroyReadH(&pQueryHandle->rhelper); tdFreeDataCols(pQueryHandle->pDataCols); pQueryHandle->pDataCols = NULL; + pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); + pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + SIOCostSummary* pCost = &pQueryHandle->cost; - tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", - pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); + tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %"PRIu64, + pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); tfree(pQueryHandle); } diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c new file mode 100644 index 0000000000000000000000000000000000000000..7212ae1636400b0ce45ef27306356f658fb3f90d --- /dev/null +++ b/src/tsdb/src/tsdbReadImpl.c @@ -0,0 +1,660 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdbint.h" + +#define TSDB_KEY_COL_OFFSET 0 + +static void tsdbResetReadTable(SReadH *pReadh); +static void tsdbResetReadFile(SReadH *pReadh); +static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, + int maxPoints, char *buffer, int bufferSize); +static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds, + int numOfColIds); +static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol); + +int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo) { + ASSERT(pReadh != NULL && pRepo != NULL); + + STsdbCfg *pCfg = REPO_CFG(pRepo); + + memset((void *)pReadh, 0, sizeof(*pReadh)); + pReadh->pRepo = pRepo; + + TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); + + pReadh->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx)); + if (pReadh->aBlkIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + pReadh->pDCols[0] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + if (pReadh->pDCols[0] == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyReadH(pReadh); + return -1; + } + + pReadh->pDCols[1] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + if (pReadh->pDCols[1] == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbDestroyReadH(pReadh); + return -1; + } + + return 0; +} + +void tsdbDestroyReadH(SReadH *pReadh) { + if (pReadh == NULL) return; + + pReadh->pCBuf = taosTZfree(pReadh->pCBuf); + pReadh->pBuf = taosTZfree(pReadh->pBuf); + pReadh->pDCols[0] = tdFreeDataCols(pReadh->pDCols[0]); + pReadh->pDCols[1] = tdFreeDataCols(pReadh->pDCols[1]); + pReadh->pBlkData = taosTZfree(pReadh->pBlkData); + pReadh->pBlkInfo = taosTZfree(pReadh->pBlkInfo); + pReadh->cidx = 0; + pReadh->pBlkIdx = NULL; + pReadh->pTable = NULL; + pReadh->aBlkIdx = taosArrayDestroy(pReadh->aBlkIdx); + tsdbCloseDFileSet(TSDB_READ_FSET(pReadh)); + pReadh->pRepo = NULL; +} + +int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) { + ASSERT(pSet != NULL); + tsdbResetReadFile(pReadh); + + pReadh->rSet = *pSet; + TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); + if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { + tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet), + tstrerror(terrno)); + return -1; + } + + return 0; +} + +void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); } + +int tsdbLoadBlockIdx(SReadH *pReadh) { + SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); + SBlockIdx blkIdx; + + ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0); + + // No data at all, just return + if (pHeadf->info.offset <= 0) return 0; + + if (tsdbSeekDFile(pHeadf, pHeadf->info.offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load SBlockIdx part while seek file %s since %s, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset, + pHeadf->info.len); + return -1; + } + + if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pHeadf->info.len) < 0) return -1; + + int64_t nread = tsdbReadDFile(pHeadf, TSDB_READ_BUF(pReadh), pHeadf->info.len); + if (nread < 0) { + tsdbError("vgId:%d failed to load SBlockIdx part while read file %s since %s, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset, + pHeadf->info.len); + return -1; + } + + if (nread < pHeadf->info.len) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d SBlockIdx part in file %s is corrupted, offset:%u expected bytes:%u read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len, nread); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), pHeadf->info.len)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d SBlockIdx part in file %s is corrupted since wrong checksum, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len); + return -1; + } + + void *ptr = TSDB_READ_BUF(pReadh); + int tsize = 0; + while (POINTER_DISTANCE(ptr, TSDB_READ_BUF(pReadh)) < (pHeadf->info.len - sizeof(TSCKSUM))) { + ptr = tsdbDecodeSBlockIdx(ptr, &blkIdx); + ASSERT(ptr != NULL); + + if (taosArrayPush(pReadh->aBlkIdx, (void *)(&blkIdx)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + tsize++; + ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid < + ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid); + } + + return 0; +} + +int tsdbSetReadTable(SReadH *pReadh, STable *pTable) { + STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); + + pReadh->pTable = pTable; + + if (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + if (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + size_t size = taosArrayGetSize(pReadh->aBlkIdx); + if (size > 0) { + while (true) { + if (pReadh->cidx >= size) { + pReadh->pBlkIdx = NULL; + break; + } + + SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx); + if (pBlkIdx->tid == TABLE_TID(pTable)) { + if (pBlkIdx->uid == TABLE_UID(pTable)) { + pReadh->pBlkIdx = pBlkIdx; + } else { + pReadh->pBlkIdx = NULL; + } + pReadh->cidx++; + break; + } else if (pBlkIdx->tid > TABLE_TID(pTable)) { + pReadh->pBlkIdx = NULL; + break; + } else { + pReadh->cidx++; + } + } + } else { + pReadh->pBlkIdx = NULL; + } + + return 0; +} + +int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) { + ASSERT(pReadh->pBlkIdx != NULL); + + SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); + SBlockIdx *pBlkIdx = pReadh->pBlkIdx; + + if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len); + return -1; + } + + if (tsdbMakeRoom((void **)(&(pReadh->pBlkInfo)), pBlkIdx->len) < 0) return -1; + + int64_t nread = tsdbReadDFile(pHeadf, (void *)(pReadh->pBlkInfo), pBlkIdx->len); + if (nread < 0) { + tsdbError("vgId:%d failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len); + return -1; + } + + if (nread < pBlkIdx->len) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, nread); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkInfo), pBlkIdx->len)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len); + return -1; + } + + ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid); + + if (pTarget) { + memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len); + } + + return 0; +} + +int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { + ASSERT(pBlock->numOfSubBlocks > 0); + + SBlock *iBlock = pBlock; + if (pBlock->numOfSubBlocks > 1) { + if (pBlkInfo) { + iBlock = (SBlock *)POINTER_SHIFT(pBlkInfo, pBlock->offset); + } else { + iBlock = (SBlock *)POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset); + } + } + + if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1; + for (int i = 1; i < pBlock->numOfSubBlocks; i++) { + iBlock++; + if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1; + if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows) < 0) return -1; + } + + ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); + ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); + + return 0; +} + +int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds) { + ASSERT(pBlock->numOfSubBlocks > 0); + + SBlock *iBlock = pBlock; + if (pBlock->numOfSubBlocks > 1) { + if (pBlkInfo) { + iBlock = POINTER_SHIFT(pBlkInfo, pBlock->offset); + } else { + iBlock = POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset); + } + } + + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1; + for (int i = 1; i < pBlock->numOfSubBlocks; i++) { + iBlock++; + if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1; + if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows) < 0) return -1; + } + + ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); + ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); + ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); + + return 0; +} + +int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { + ASSERT(pBlock->numOfSubBlocks <= 1); + + SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + + if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); + return -1; + } + + size_t size = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols); + if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1; + + int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); + if (nread < 0) { + tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size); + return -1; + } + + if (nread < size) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu + " read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size); + return -1; + } + + return 0; +} + +int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) { + int tlen = 0; + + tlen += taosEncodeVariantI32(buf, pIdx->tid); + tlen += taosEncodeVariantU32(buf, pIdx->len); + tlen += taosEncodeVariantU32(buf, pIdx->offset); + tlen += taosEncodeFixedU8(buf, pIdx->hasLast); + tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks); + tlen += taosEncodeFixedU64(buf, pIdx->uid); + tlen += taosEncodeFixedU64(buf, pIdx->maxKey); + + return tlen; +} + +void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) { + uint8_t hasLast = 0; + uint32_t numOfBlocks = 0; + uint64_t value = 0; + + if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; + if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; + pIdx->hasLast = hasLast; + if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; + pIdx->numOfBlocks = numOfBlocks; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; + pIdx->uid = (int64_t)value; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; + pIdx->maxKey = (TSKEY)value; + + return buf; +} + +void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols) { + SBlockData *pBlockData = pReadh->pBlkData; + + for (int i = 0, j = 0; i < numOfCols;) { + if (j >= pBlockData->numOfCols) { + pStatis[i].numOfNull = -1; + i++; + continue; + } + + if (pStatis[i].colId == pBlockData->cols[j].colId) { + pStatis[i].sum = pBlockData->cols[j].sum; + pStatis[i].max = pBlockData->cols[j].max; + pStatis[i].min = pBlockData->cols[j].min; + pStatis[i].maxIndex = pBlockData->cols[j].maxIndex; + pStatis[i].minIndex = pBlockData->cols[j].minIndex; + pStatis[i].numOfNull = pBlockData->cols[j].numOfNull; + i++; + j++; + } else if (pStatis[i].colId < pBlockData->cols[j].colId) { + pStatis[i].numOfNull = -1; + i++; + } else { + j++; + } + } +} + +static void tsdbResetReadTable(SReadH *pReadh) { + tdResetDataCols(pReadh->pDCols[0]); + tdResetDataCols(pReadh->pDCols[1]); + pReadh->cidx = 0; + pReadh->pBlkIdx = NULL; + pReadh->pTable = NULL; +} + +static void tsdbResetReadFile(SReadH *pReadh) { + tsdbResetReadTable(pReadh); + taosArrayClear(pReadh->aBlkIdx); + tsdbCloseDFileSet(TSDB_READ_FSET(pReadh)); +} + +static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) { + ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); + + SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + + tdResetDataCols(pDataCols); + if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1; + + SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh); + + if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block data part while seek file %s to offset %" PRId64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); + return -1; + } + + int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlock->len); + if (nread < 0) { + tsdbError("vgId:%d failed to load block data part while read file %s since %s, offset:%" PRId64 " len :%d", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, + pBlock->len); + return -1; + } + + if (nread < pBlock->len) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block data part in file %s is corrupted, offset:%" PRId64 + " expected bytes:%d read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, pBlock->len, nread); + return -1; + } + + int32_t tsize = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols); + if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize); + return -1; + } + + ASSERT(tsize < pBlock->len); + ASSERT(pBlockData->numOfCols == pBlock->numOfCols); + + pDataCols->numOfRows = pBlock->numOfRows; + + // Recover the data + int ccol = 0; // loop iter for SBlockCol object + int dcol = 0; // loop iter for SDataCols object + while (dcol < pDataCols->numOfCols) { + SDataCol *pDataCol = &(pDataCols->cols[dcol]); + if (dcol != 0 && ccol >= pBlockData->numOfCols) { + // Set current column as NULL and forward + dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + dcol++; + continue; + } + + int16_t tcolId = 0; + uint32_t toffset = TSDB_KEY_COL_OFFSET; + int32_t tlen = pBlock->keyLen; + + if (dcol != 0) { + SBlockCol *pBlockCol = &(pBlockData->cols[ccol]); + tcolId = pBlockCol->colId; + toffset = tsdbGetBlockColOffset(pBlockCol); + tlen = pBlockCol->len; + } else { + ASSERT(pDataCol->colId == tcolId); + } + + if (tcolId == pDataCol->colId) { + if (pBlock->algorithm == TWO_STAGE_COMP) { + int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES; + if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1; + } + + if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm, + pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), + (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { + tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); + return -1; + } + + if (dcol != 0) { + ccol++; + } + dcol++; + } else if (tcolId < pDataCol->colId) { + ccol++; + } else { + // Set current column as NULL and forward + dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + dcol++; + } + } + + return 0; +} + +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, + int maxPoints, char *buffer, int bufferSize) { + if (!taosCheckChecksumWhole((uint8_t *)content, len)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + // Decode the data + if (comp) { + // Need to decompress + int tlen = (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, + pDataCol->spaceSize, comp, buffer, bufferSize); + if (tlen <= 0) { + tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", + len, comp, numOfRows, maxPoints, bufferSize); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + pDataCol->len = tlen; + } else { + // No need to decompress, just memcpy it + pDataCol->len = len - sizeof(TSCKSUM); + memcpy(pDataCol->pData, content, pDataCol->len); + } + + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + dataColSetOffset(pDataCol, numOfRows); + } + return 0; +} + +static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds, + int numOfColIds) { + ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1); + ASSERT(colIds[0] == 0); + + SDFile * pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); + SBlockCol blockCol = {0}; + + tdResetDataCols(pDataCols); + + // If only load timestamp column, no need to load SBlockData part + if (numOfColIds > 1 && tsdbLoadBlockStatis(pReadh, pBlock) < 0) return -1; + + pDataCols->numOfRows = pBlock->numOfRows; + + int dcol = 0; + int ccol = 0; + for (int i = 0; i < numOfColIds; i++) { + int16_t colId = colIds[i]; + SDataCol * pDataCol = NULL; + SBlockCol *pBlockCol = NULL; + + while (true) { + if (dcol >= pDataCols->numOfCols) { + pDataCol = NULL; + break; + } + pDataCol = &pDataCols->cols[dcol]; + if (pDataCol->colId > colId) { + pDataCol = NULL; + break; + } else { + dcol++; + if (pDataCol->colId == colId) break; + } + } + + if (pDataCol == NULL) continue; + ASSERT(pDataCol->colId == colId); + + if (colId == 0) { // load the key row + blockCol.colId = colId; + blockCol.len = pBlock->keyLen; + blockCol.type = pDataCol->type; + blockCol.offset = TSDB_KEY_COL_OFFSET; + pBlockCol = &blockCol; + } else { // load non-key rows + while (true) { + if (ccol >= pBlock->numOfCols) { + pBlockCol = NULL; + break; + } + + pBlockCol = &(pReadh->pBlkData->cols[ccol]); + if (pBlockCol->colId > colId) { + pBlockCol = NULL; + break; + } else { + ccol++; + if (pBlockCol->colId == colId) break; + } + } + + if (pBlockCol == NULL) { + dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + continue; + } + + ASSERT(pBlockCol->colId == pDataCol->colId); + } + + if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1; + } + + return 0; +} + +static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) { + ASSERT(pDataCol->colId == pBlockCol->colId); + + STsdbRepo *pRepo = TSDB_READ_REPO(pReadh); + STsdbCfg * pCfg = REPO_CFG(pRepo); + int tsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES; + + if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; + if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; + + int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + tsdbGetBlockColOffset(pBlockCol); + if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno)); + return -1; + } + + int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlockCol->len); + if (nread < 0) { + tsdbError("vgId:%d failed to load block column data while read file %s since %s, offset:%" PRId64 " len :%d", + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), offset, pBlockCol->len); + return -1; + } + + if (nread < pBlockCol->len) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + tsdbError("vgId:%d block column data in file %s is corrupted, offset:%" PRId64 " expected bytes:%d" PRIzu + " read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, pBlockCol->len, nread); + return -1; + } + + if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows, + pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) { + tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), + pBlockCol->colId, offset); + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbScan.c b/src/tsdb/src/tsdbScan.c index 91f67878740c90d56a8c5c124be23c1f53e7da40..382f7b11ae021152c9c8d314b24428b51c2e107b 100644 --- a/src/tsdb/src/tsdbScan.c +++ b/src/tsdb/src/tsdbScan.c @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -#include "tsdbMain.h" +#include "tsdbint.h" +#if 0 #ifndef _TSDB_PLUGINS int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } @@ -25,12 +26,13 @@ void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream) {} int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } -int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle) { return 0; } +int tsdbScanSBlockIdx(STsdbScanHandle* pScanHandle) { return 0; } -int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; } +int tsdbScanSBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; } int tsdbCloseScanFile(STsdbScanHandle* pScanHandle) { return 0; } void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle) {} +#endif #endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbSync.c b/src/tsdb/src/tsdbSync.c new file mode 100644 index 0000000000000000000000000000000000000000..5a2756537e6a37c8b4a0e7c3e1f81199523df2eb --- /dev/null +++ b/src/tsdb/src/tsdbSync.c @@ -0,0 +1,699 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "tsdbint.h" + +// Sync handle +typedef struct { + STsdbRepo *pRepo; + SRtn rtn; + SOCKET socketFd; + void * pBuf; + bool mfChanged; + SMFile * pmf; + SMFile mf; + SDFileSet df; + SDFileSet *pdf; +} SSyncH; + +#define SYNC_BUFFER(sh) ((sh)->pBuf) + +static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd); +static void tsdbDestroySyncH(SSyncH *pSyncH); +static int32_t tsdbSyncSendMeta(SSyncH *pSynch); +static int32_t tsdbSyncRecvMeta(SSyncH *pSynch); +static int32_t tsdbSendMetaInfo(SSyncH *pSynch); +static int32_t tsdbRecvMetaInfo(SSyncH *pSynch); +static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend); +static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend); +static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch); +static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch); +static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2); +static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet); +static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet); +static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch); +static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged); + +int32_t tsdbSyncSend(void *tsdb, SOCKET socketFd) { + STsdbRepo *pRepo = (STsdbRepo *)tsdb; + SSyncH synch = {0}; + + tsdbInitSyncH(&synch, pRepo, socketFd); + // Disable TSDB commit + tsem_wait(&(pRepo->readyToCommit)); + + if (tsdbSyncSendMeta(&synch) < 0) { + tsdbError("vgId:%d, failed to send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + if (tsdbSyncSendDFileSetArray(&synch) < 0) { + tsdbError("vgId:%d, failed to send filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + // Enable TSDB commit + tsem_post(&(pRepo->readyToCommit)); + tsdbDestroySyncH(&synch); + return 0; + +_err: + tsem_post(&(pRepo->readyToCommit)); + tsdbDestroySyncH(&synch); + return -1; +} + +int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) { + STsdbRepo *pRepo = (STsdbRepo *)tsdb; + SSyncH synch = {0}; + + pRepo->state = TSDB_STATE_OK; + + tsdbInitSyncH(&synch, pRepo, socketFd); + tsem_wait(&(pRepo->readyToCommit)); + tsdbStartFSTxn(pRepo, 0, 0); + + if (tsdbSyncRecvMeta(&synch) < 0) { + tsdbError("vgId:%d, failed to recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + if (tsdbSyncRecvDFileSetArray(&synch) < 0) { + tsdbError("vgId:%d, failed to recv filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + tsdbEndFSTxn(pRepo); + tsem_post(&(pRepo->readyToCommit)); + tsdbDestroySyncH(&synch); + + // Reload file change + tsdbReload(pRepo, synch.mfChanged); + + return 0; + +_err: + tsdbEndFSTxnWithError(REPO_FS(pRepo)); + tsem_post(&(pRepo->readyToCommit)); + tsdbDestroySyncH(&synch); + return -1; +} + +static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd) { + pSyncH->pRepo = pRepo; + pSyncH->socketFd = socketFd; + tsdbGetRtnSnap(pRepo, &(pSyncH->rtn)); +} + +static void tsdbDestroySyncH(SSyncH *pSyncH) { taosTZfree(pSyncH->pBuf); } + +static int32_t tsdbSyncSendMeta(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + bool toSendMeta = false; + SMFile mf; + + // Send meta info to remote + tsdbInfo("vgId:%d, metainfo will be sent", REPO_ID(pRepo)); + if (tsdbSendMetaInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (pRepo->fs->cstatus->pmf == NULL) { + // No meta file, not need to wait to retrieve meta file + tsdbInfo("vgId:%d, metafile not exist, no need to send", REPO_ID(pRepo)); + return 0; + } + + if (tsdbRecvDecision(pSynch, &toSendMeta) < 0) { + tsdbError("vgId:%d, failed to recv decision while send meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (toSendMeta) { + tsdbInitMFileEx(&mf, pRepo->fs->cstatus->pmf); + if (tsdbOpenMFile(&mf, O_RDONLY) < 0) { + tsdbError("vgId:%d, failed to open file while send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + int64_t writeLen = mf.info.size; + tsdbInfo("vgId:%d, metafile:%s will be sent, size:%" PRId64, REPO_ID(pRepo), mf.f.aname, writeLen); + + int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&mf), 0, writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send metafile since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), + tstrerror(terrno), ret, writeLen); + tsdbCloseMFile(&mf); + return -1; + } + + tsdbCloseMFile(&mf); + tsdbInfo("vgId:%d, metafile is sent", REPO_ID(pRepo)); + } else { + tsdbInfo("vgId:%d, metafile is same, no need to send", REPO_ID(pRepo)); + } + + return 0; +} + +static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + SMFile * pLMFile = pRepo->fs->cstatus->pmf; + + // Recv meta info from remote + if (tsdbRecvMetaInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + // No meta file, do nothing (rm local meta file) + if (pSynch->pmf == NULL) { + if (pLMFile == NULL) { + pSynch->mfChanged = false; + } else { + pSynch->mfChanged = true; + } + tsdbInfo("vgId:%d, metafile not exist in remote, no need to recv", REPO_ID(pRepo)); + return 0; + } + + if (pLMFile == NULL || pSynch->pmf->info.size != pLMFile->info.size || + pSynch->pmf->info.magic != pLMFile->info.magic || TSDB_FILE_IS_BAD(pLMFile)) { + // Local has no meta file or has a different meta file, need to copy from remote + pSynch->mfChanged = true; + + if (tsdbSendDecision(pSynch, true) < 0) { + tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d, metafile will be received", REPO_ID(pRepo)); + + // Recv from remote + SMFile mf; + SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID}; + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + if (tsdbCreateMFile(&mf, false) < 0) { + tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d, metafile:%s is created", REPO_ID(pRepo), mf.f.aname); + + int64_t readLen = pSynch->pmf->info.size; + int64_t ret = taosCopyFds(pSynch->socketFd, TSDB_FILE_FD(&mf), readLen); + if (ret != readLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv metafile since %s, ret:%" PRId64 " readLen:%" PRId64, REPO_ID(pRepo), + tstrerror(terrno), ret, readLen); + tsdbCloseMFile(&mf); + tsdbRemoveMFile(&mf); + return -1; + } + + tsdbInfo("vgId:%d, metafile is received, size:%" PRId64, REPO_ID(pRepo), readLen); + + mf.info = pSynch->pmf->info; + tsdbCloseMFile(&mf); + tsdbUpdateMFile(REPO_FS(pRepo), &mf); + } else { + pSynch->mfChanged = false; + tsdbInfo("vgId:%d, metafile is same, no need to recv", REPO_ID(pRepo)); + if (tsdbSendDecision(pSynch, false) < 0) { + tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + tsdbUpdateMFile(REPO_FS(pRepo), pLMFile); + } + + return 0; +} + +static int32_t tsdbSendMetaInfo(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + uint32_t tlen = 0; + SMFile * pMFile = pRepo->fs->cstatus->pmf; + + if (pMFile) { + tlen = tlen + tsdbEncodeSMFileEx(NULL, pMFile) + sizeof(TSCKSUM); + } + + if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { + tsdbError("vgId:%d, failed to makeroom while send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + void *ptr = SYNC_BUFFER(pSynch); + taosEncodeFixedU32(&ptr, tlen); + void *tptr = ptr; + if (pMFile) { + tsdbEncodeSMFileEx(&ptr, pMFile); + taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); + } + + int32_t writeLen = tlen + sizeof(uint32_t); + int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send metainfo since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret, + writeLen); + return -1; + } + + tsdbInfo("vgId:%d, metainfo is sent, tlen:%d, writeLen:%d", REPO_ID(pRepo), tlen, writeLen); + return 0; +} + +static int32_t tsdbRecvMetaInfo(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + uint32_t tlen = 0; + char buf[64] = {0}; + + int32_t readLen = sizeof(uint32_t); + int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); + if (ret != readLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv metalen, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); + return -1; + } + + taosDecodeFixedU32(buf, &tlen); + + tsdbInfo("vgId:%d, metalen is received, readLen:%d, tlen:%d", REPO_ID(pRepo), readLen, tlen); + if (tlen == 0) { + pSynch->pmf = NULL; + return 0; + } + + if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { + tsdbError("vgId:%d, failed to makeroom while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); + if (ret != tlen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv metainfo, ret:%d tlen:%d", REPO_ID(pRepo), ret, tlen); + return -1; + } + + tsdbInfo("vgId:%d, metainfo is received, tlen:%d", REPO_ID(pRepo), tlen); + if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { + terrno = TSDB_CODE_TDB_MESSED_MSG; + tsdbError("vgId:%d, failed to checksum while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + pSynch->pmf = &(pSynch->mf); + tsdbDecodeSMFileEx(SYNC_BUFFER(pSynch), pSynch->pmf); + + return 0; +} + +static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend) { + STsdbRepo *pRepo = pSynch->pRepo; + uint8_t decision = toSend; + + int32_t writeLen = sizeof(uint8_t); + int32_t ret = taosWriteMsg(pSynch->socketFd, (void *)(&decision), writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send decison, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); + return -1; + } + + return 0; +} + +static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend) { + STsdbRepo *pRepo = pSynch->pRepo; + uint8_t decision = 0; + + int32_t readLen = sizeof(uint8_t); + int32_t ret = taosReadMsg(pSynch->socketFd, (void *)(&decision), readLen); + if (ret != readLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv decison, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); + return -1; + } + + *toSend = decision; + return 0; +} + +static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + STsdbFS * pfs = REPO_FS(pRepo); + SFSIter fsiter; + SDFileSet *pSet; + + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + + do { + pSet = tsdbFSIterNext(&fsiter); + if (tsdbSyncSendDFileSet(pSynch, pSet) < 0) { + tsdbError("vgId:%d, failed to send fileset:%d since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, + tstrerror(terrno)); + return -1; + } + + // No more file set to send, jut break + if (pSet == NULL) { + tsdbInfo("vgId:%d, no filesets any more", REPO_ID(pRepo)); + break; + } + } while (true); + + return 0; +} + +static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + STsdbFS * pfs = REPO_FS(pRepo); + SFSIter fsiter; + SDFileSet *pLSet; // Local file set + + tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); + + pLSet = tsdbFSIterNext(&fsiter); + if (tsdbRecvDFileSetInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + while (true) { + if (pLSet == NULL && pSynch->pdf == NULL) { + tsdbInfo("vgId:%d, all filesets is disposed", REPO_ID(pRepo)); + break; + } else { + tsdbInfo("vgId:%d, fileset local:%d remote:%d, will be disposed", REPO_ID(pRepo), pLSet != NULL ? pLSet->fid : -1, + pSynch->pdf != NULL ? pSynch->pdf->fid : -1); + } + + if (pLSet && (pSynch->pdf == NULL || pLSet->fid < pSynch->pdf->fid)) { + // remote not has pLSet->fid set, just remove local (do nothing to remote the fset) + tsdbInfo("vgId:%d, fileset:%d smaller than remote:%d, remove it", REPO_ID(pRepo), pLSet->fid, + pSynch->pdf != NULL ? pSynch->pdf->fid : -1); + pLSet = tsdbFSIterNext(&fsiter); + } else { + if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf) && + tsdbFSetIsOk(pLSet)) { + // Just keep local files and notify remote not to send + tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid); + + if (tsdbUpdateDFileSet(pfs, pLSet) < 0) { + tsdbError("vgId:%d, failed to update fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (tsdbSendDecision(pSynch, false) < 0) { + tsdbError("vgId:%d, filed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } else { + // Need to copy from remote + tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid); + + // Notify remote to send there file here + if (tsdbSendDecision(pSynch, true) < 0) { + tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + // Create local files and copy from remote + SDiskID did; + SDFileSet fset; + + tfsAllocDisk(tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)), &(did.level), &(did.id)); + if (did.level == TFS_UNDECIDED_LEVEL) { + terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; + tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInitDFileSet(&fset, did, REPO_ID(pRepo), pSynch->pdf->fid, FS_TXN_VERSION(pfs)); + + // Create new FSET + if (tsdbCreateDFileSet(&fset, false) < 0) { + tsdbError("vgId:%d, failed to create fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); // local file + SDFile *pRDFile = TSDB_DFILE_IN_SET(pSynch->pdf, ftype); // remote file + + tsdbInfo("vgId:%d, file:%s will be received, osize:%" PRIu64 " rsize:%" PRIu64, REPO_ID(pRepo), + pDFile->f.aname, pDFile->info.size, pRDFile->info.size); + + int64_t writeLen = pRDFile->info.size; + int64_t ret = taosCopyFds(pSynch->socketFd, pDFile->fd, writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), + pDFile->f.aname, tstrerror(terrno), ret, writeLen); + tsdbCloseDFileSet(&fset); + tsdbRemoveDFileSet(&fset); + return -1; + } + + // Update new file info + pDFile->info = pRDFile->info; + tsdbInfo("vgId:%d, file:%s is received, size:%" PRId64, REPO_ID(pRepo), pDFile->f.aname, writeLen); + } + + tsdbCloseDFileSet(&fset); + if (tsdbUpdateDFileSet(pfs, &fset) < 0) { + tsdbInfo("vgId:%d, fileset:%d failed to update since %s", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d, fileset:%d is received", REPO_ID(pRepo), pSynch->pdf->fid); + } + + // Move forward + if (tsdbRecvDFileSetInfo(pSynch) < 0) { + tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + if (pLSet) { + pLSet = tsdbFSIterNext(&fsiter); + } + } + +#if 0 + if (pLSet == NULL) { + // Copy from remote >>>>>>>>>>> + } else { + if (pSynch->pdf == NULL) { + // Remove local file, just ignore ++++++++++++++ + pLSet = tsdbFSIterNext(&fsiter); + } else { + if (pLSet->fid < pSynch->pdf->fid) { + // Remove local file, just ignore ++++++++++++ + pLSet = tsdbFSIterNext(&fsiter); + } else if (pLSet->fid > pSynch->pdf->fid){ + // Copy from remote >>>>>>>>>>>>>> + if (tsdbRecvDFileSetInfo(pSynch) < 0) { + // TODO + return -1; + } + } else { + if (true/*TODO: is same fset*/) { + // No need to copy --------------------- + } else { + // copy from remote >>>>>>>>>>>>>. + } + } + } + } +#endif + } + + return 0; +} + +static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype); + SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype); + + if (pDFile1->info.size != pDFile2->info.size || pDFile1->info.magic != pDFile2->info.magic) { + return false; + } + } + + return true; +} + +static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) { + STsdbRepo *pRepo = pSynch->pRepo; + bool toSend = false; + + if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) { + tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, tstrerror(terrno)); + return -1; + } + + // No file any more, no need to send file, just return + if (pSet == NULL) { + return 0; + } + + if (tsdbRecvDecision(pSynch, &toSend) < 0) { + tsdbError("vgId:%d, failed to recv decision while send fileset:%d since %s", REPO_ID(pRepo), pSet->fid, + tstrerror(terrno)); + return -1; + } + + if (toSend) { + tsdbInfo("vgId:%d, fileset:%d will be sent", REPO_ID(pRepo), pSet->fid); + + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile df = *TSDB_DFILE_IN_SET(pSet, ftype); + + if (tsdbOpenDFile(&df, O_RDONLY) < 0) { + tsdbError("vgId:%d, failed to file:%s since %s", REPO_ID(pRepo), df.f.aname, tstrerror(terrno)); + return -1; + } + + int64_t writeLen = df.info.size; + tsdbInfo("vgId:%d, file:%s will be sent, size:%" PRId64, REPO_ID(pRepo), df.f.aname, writeLen); + + int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&df), 0, writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), + df.f.aname, tstrerror(terrno), ret, writeLen); + tsdbCloseDFile(&df); + return -1; + } + + tsdbInfo("vgId:%d, file:%s is sent", REPO_ID(pRepo), df.f.aname); + tsdbCloseDFile(&df); + } + + tsdbInfo("vgId:%d, fileset:%d is sent", REPO_ID(pRepo), pSet->fid); + } else { + tsdbInfo("vgId:%d, fileset:%d is same, no need to send", REPO_ID(pRepo), pSet->fid); + } + + return 0; +} + +static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet) { + STsdbRepo *pRepo = pSynch->pRepo; + uint32_t tlen = 0; + + if (pSet) { + tlen = tsdbEncodeDFileSetEx(NULL, pSet) + sizeof(TSCKSUM); + } + + if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { + tsdbError("vgId:%d, failed to makeroom while send fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + void *ptr = SYNC_BUFFER(pSynch); + taosEncodeFixedU32(&ptr, tlen); + void *tptr = ptr; + if (pSet) { + tsdbEncodeDFileSetEx(&ptr, pSet); + taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); + } + + int32_t writeLen = tlen + sizeof(uint32_t); + int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); + if (ret != writeLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to send fileinfo, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); + return -1; + } + + return 0; +} + +static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch) { + STsdbRepo *pRepo = pSynch->pRepo; + uint32_t tlen; + char buf[64] = {0}; + + int32_t readLen = sizeof(uint32_t); + int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); + if (ret != readLen) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + taosDecodeFixedU32(buf, &tlen); + + tsdbInfo("vgId:%d, fileinfo len:%d is received", REPO_ID(pRepo), tlen); + if (tlen == 0) { + pSynch->pdf = NULL; + return 0; + } + + if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { + tsdbError("vgId:%d, failed to makeroom while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); + if (ret != tlen) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbError("vgId:%d, failed to recv fileinfo, ret:%d readLen:%d", REPO_ID(pRepo), ret, tlen); + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { + terrno = TSDB_CODE_TDB_MESSED_MSG; + tsdbError("vgId:%d, failed to checksum while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + pSynch->pdf = &(pSynch->df); + tsdbDecodeDFileSetEx(SYNC_BUFFER(pSynch), pSynch->pdf); + + return 0; +} + +static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged) { + // TODO: may need to stop and restart stream + // if (isMfChanged) { + tsdbCloseMeta(pRepo); + tsdbFreeMeta(pRepo->tsdbMeta); + pRepo->tsdbMeta = tsdbNewMeta(REPO_CFG(pRepo)); + tsdbOpenMeta(pRepo); + tsdbLoadMetaCache(pRepo, true); + // } + + tsdbUnRefMemTable(pRepo, pRepo->mem); + tsdbUnRefMemTable(pRepo, pRepo->imem); + pRepo->mem = NULL; + pRepo->imem = NULL; + + if (tsdbRestoreInfo(pRepo) < 0) { + tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + return 0; +} \ No newline at end of file diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index ef5ed6f04459a4213e761f94ad00363ede9ecd26..ac254d6c34ecd152e28bb621348312b938fecd20 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -12,7 +12,7 @@ static double getCurTime() { } typedef struct { - TSDB_REPO_T *pRepo; + STsdbRepo *pRepo; bool isAscend; int tid; uint64_t uid; @@ -143,7 +143,7 @@ TEST(TsdbTest, testInsertSpeed) { // Create and open repository tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1); tsdbCreateRepo(rootDir, &tsdbCfg); - TSDB_REPO_T *repo = tsdbOpenRepo(rootDir, NULL); + STsdbRepo *repo = tsdbOpenRepo(rootDir, NULL); ASSERT_NE(repo, nullptr); // Create table diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 48b4d7656105aed1a983fad59c20195f0d3fcb77..a7f4f59e07021b659707c59a0c9b2ef916558d52 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -1,11 +1,13 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) -TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z) - +TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4 z rmonotonic) + IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) # ADD_SUBDIRECTORY(tests) @@ -28,5 +30,10 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) TARGET_LINK_LIBRARIES(tutil iconv regex winmm IPHLPAPI ws2_32 wepoll) ELSEIF(TD_DARWIN) + TARGET_LINK_LIBRARIES(tutil m) TARGET_LINK_LIBRARIES(tutil iconv) ENDIF() + +IF (TD_STORAGE) + TARGET_LINK_LIBRARIES(tutil storage) +ENDIF () \ No newline at end of file diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index bf922fe9c44b4f923d44f4a38c1c4c1a7fb20af3..f2e268c2d4dfe210dfbfd9b94ee74a4f87848361 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -21,9 +21,12 @@ extern "C" { #endif #include "os.h" +#include "talgo.h" #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) +#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) +#define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { size_t size; @@ -44,9 +47,20 @@ void* taosArrayInit(size_t size, size_t elemSize); * * @param pArray * @param pData + * @param nEles * @return */ -void* taosArrayPush(SArray* pArray, void* pData); +void *taosArrayPushBatch(SArray *pArray, const void *pData, int nEles); + +/** + * + * @param pArray + * @param pData + * @return + */ +static FORCE_INLINE void* taosArrayPush(SArray* pArray, const void* pData) { + return taosArrayPushBatch(pArray, pData, 1); +} /** * @@ -92,6 +106,14 @@ size_t taosArrayGetSize(const SArray* pArray); */ void* taosArrayInsert(SArray* pArray, size_t index, void* pData); +/** + * set data in array + * @param pArray + * @param index + * @param pData + */ +void taosArraySet(SArray* pArray, size_t index, void* pData); + /** * remove data entry of the given index * @param pArray @@ -104,13 +126,13 @@ void taosArrayRemove(SArray* pArray, size_t index); * @param pDst * @param pSrc */ -void taosArrayCopy(SArray* pDst, const SArray* pSrc); +SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize); /** * clone a new array * @param pSrc */ -SArray* taosArrayClone(const SArray* pSrc); +SArray* taosArrayDup(const SArray* pSrc); /** * clear the array (remove all element) @@ -122,7 +144,7 @@ void taosArrayClear(SArray* pArray); * destroy array list * @param pArray */ -void taosArrayDestroy(SArray* pArray); +void* taosArrayDestroy(SArray* pArray); /** * @@ -136,7 +158,7 @@ void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)); * @param pArray * @param compar */ -void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)); +void taosArraySort(SArray* pArray, __compar_fn_t comparFn); /** * sort string array @@ -150,14 +172,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn); * @param compar * @param key */ -void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn); +void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); /** * search the array * @param pArray * @param key */ -char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn); +char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags); #ifdef __cplusplus } diff --git a/src/util/inc/tbuffer.h b/src/util/inc/tbuffer.h index e2bdb815d7ac8ecd09b62a3a62897b78ebf3c4b7..b19984b4be047108fd9848a2a7eddbac2c0b78a4 100644 --- a/src/util/inc/tbuffer.h +++ b/src/util/inc/tbuffer.h @@ -73,14 +73,14 @@ int main( int argc, char** argv ) { } */ -typedef struct { +typedef struct SBufferReader { bool endian; const char* data; size_t pos; size_t size; } SBufferReader; -typedef struct { +typedef struct SBufferWriter { bool endian; char* data; size_t pos; diff --git a/src/util/inc/tchecksum.h b/src/util/inc/tchecksum.h index 495aaf33e8fca3594c45f1cfe6fc6acc59fa59d5..12ca3a54432187ea5af0ac24208e81d5028359bb 100644 --- a/src/util/inc/tchecksum.h +++ b/src/util/inc/tchecksum.h @@ -47,7 +47,6 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin } static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { - if (ssize < 0) return 0; return (checksum == (*crc32c)(0, stream, (size_t)ssize)); } diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index bc1da9858a3ea66610c6d03364d753b6b2f06313..fdb2595fd8d3b1659800ca3a5b88c32b6fe95e93 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -44,10 +44,12 @@ enum { TAOS_CFG_VTYPE_INT8, TAOS_CFG_VTYPE_INT16, TAOS_CFG_VTYPE_INT32, + TAOS_CFG_VTYPE_UINT16, TAOS_CFG_VTYPE_FLOAT, TAOS_CFG_VTYPE_STRING, TAOS_CFG_VTYPE_IPSTR, TAOS_CFG_VTYPE_DIRECTORY, + TAOS_CFG_VTYPE_DATA_DIRCTORY, }; enum { diff --git a/src/util/inc/tidpool.h b/src/util/inc/tidpool.h index bf352516310a1356f37359ab3052814d726ab8f0..e4439439ced6522e26c8db4a560c50f5b0cb8a16 100644 --- a/src/util/inc/tidpool.h +++ b/src/util/inc/tidpool.h @@ -34,7 +34,7 @@ void taosIdPoolCleanUp(void *handle); int taosIdPoolNumOfUsed(void *handle); -void taosIdPoolMarkStatus(void *handle, int id); +bool taosIdPoolMarkStatus(void *handle, int id); #ifdef __cplusplus } diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h deleted file mode 100644 index b2b0ff05f58478e3778d3abab72ae3511f683aca..0000000000000000000000000000000000000000 --- a/src/util/inc/tkvstore.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_KVSTORE_H_ -#define _TD_KVSTORE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -#define KVSTORE_FILE_VERSION ((uint32_t)0) - -typedef int (*iterFunc)(void *, void *cont, int contLen); -typedef void (*afterFunc)(void *); - -typedef struct { - int64_t size; // including 512 bytes of header size - int64_t tombSize; - int64_t nRecords; - int64_t nDels; - uint32_t magic; -} SStoreInfo; - -typedef struct { - char * fname; - int fd; - char * fsnap; - int sfd; - char * fnew; - int nfd; - SHashObj * map; - iterFunc iFunc; - afterFunc aFunc; - void * appH; - SStoreInfo info; -} SKVStore; - -#define KVSTORE_MAGIC(s) (s)->info.magic - -int tdCreateKVStore(char *fname); -int tdDestroyKVStore(char *fname); -SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); -void tdCloseKVStore(SKVStore *pStore); -int tdKVStoreStartCommit(SKVStore *pStore); -int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen); -int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid); -int tdKVStoreEndCommit(SKVStore *pStore); -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size); - -#ifdef __cplusplus -} -#endif - -#endif \ No newline at end of file diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h index e8380294da47de1c9eb65059263ac820f3d816be..6c96ec0b138cfc229ab4a8bde3f3b374bce49dfd 100644 --- a/src/util/inc/tlist.h +++ b/src/util/inc/tlist.h @@ -47,7 +47,7 @@ typedef struct { #define listNodeFree(n) free(n); SList * tdListNew(int eleSize); -void tdListFree(SList *list); +void * tdListFree(SList *list); void tdListEmpty(SList *list); void tdListPrependNode(SList *list, SListNode *node); void tdListAppendNode(SList *list, SListNode *node); diff --git a/src/util/inc/tsocket.h b/src/util/inc/tsocket.h index a339955cc02f31607c63b5c473c57c4d0e6f6f1c..b4f55169594589b83c299694723cb234bd21ed7d 100644 --- a/src/util/inc/tsocket.h +++ b/src/util/inc/tsocket.h @@ -20,11 +20,15 @@ extern "C" { #endif +#ifdef WINDOWS +#include "wepoll.h" +#endif + int32_t taosReadn(SOCKET sock, char *buffer, int32_t len); int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes); int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes); int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes); -int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len); +int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len); int32_t taosSetNonblocking(SOCKET sock, int32_t on); SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index b36d0017e84e3207a8b426e2dd91c488a539ba1f..7af03d96af4eb132288f7b8841bc26eedcf545da 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -27,6 +27,9 @@ extern "C" { #define TSQL_TBNAME "TBNAME" #define TSQL_TBNAME_L "tbname" +#define TSQL_BLOCK_DIST "_BLOCK_DIST" +#define TSQL_BLOCK_DIST_L "_block_dist" + // used to denote the minimum unite in sql parsing typedef struct SStrToken { uint32_t n; diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index bec2fac7dfd03ada681b9011df7e465cd0a49f0c..4dde5dbba24adfcda0fe794f2f36d6c059354f55 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -55,24 +55,29 @@ static int32_t taosArrayResize(SArray* pArray) { return 0; } -void* taosArrayPush(SArray* pArray, void* pData) { +void* taosArrayPushBatch(SArray* pArray, const void* pData, int nEles) { if (pArray == NULL || pData == NULL) { return NULL; } - if (pArray->size >= pArray->capacity) { - int32_t ret = taosArrayResize(pArray); - - // failed to push data into buffer due to the failure of memory allocation - if (ret != 0) { + if (pArray->size + nEles > pArray->capacity) { + size_t tsize = (pArray->capacity << 1u); + while (pArray->size + nEles > tsize) { + tsize = (tsize << 1u); + } + + pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize); + if (pArray->pData == NULL) { return NULL; } + + pArray->capacity = tsize; } void* dst = TARRAY_GET_ELEM(pArray, pArray->size); - memcpy(dst, pData, pArray->elemSize); + memcpy(dst, pData, pArray->elemSize * nEles); - pArray->size += 1; + pArray->size += nEles; return dst; } @@ -133,6 +138,11 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { return dst; } +void taosArraySet(SArray* pArray, size_t index, void* pData) { + assert(index < pArray->size); + memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize); +} + void taosArrayRemove(SArray* pArray, size_t index) { assert(index < pArray->size); @@ -146,26 +156,17 @@ void taosArrayRemove(SArray* pArray, size_t index) { pArray->size -= 1; } -void taosArrayCopy(SArray* pDst, const SArray* pSrc) { - assert(pSrc != NULL && pDst != NULL); - - if (pDst->capacity < pSrc->size) { - void* pData = realloc(pDst->pData, pSrc->size * pSrc->elemSize); - if (pData == NULL) { // todo handle oom - - } else { - pDst->pData = pData; - pDst->capacity = pSrc->size; - } - } - - memcpy(pDst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); - pDst->elemSize = pSrc->elemSize; - pDst->capacity = pSrc->size; - pDst->size = pSrc->size; +SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { + assert(src != NULL && elemSize > 0); + SArray* pDst = taosArrayInit(size, elemSize); + + memcpy(pDst->pData, src, elemSize * size); + pDst->size = size; + + return pDst; } -SArray* taosArrayClone(const SArray* pSrc) { +SArray* taosArrayDup(const SArray* pSrc) { assert(pSrc != NULL); if (pSrc->size == 0) { // empty array list @@ -184,13 +185,13 @@ void taosArrayClear(SArray* pArray) { pArray->size = 0; } -void taosArrayDestroy(SArray* pArray) { - if (pArray == NULL) { - return; +void* taosArrayDestroy(SArray* pArray) { + if (pArray) { + free(pArray->pData); + free(pArray); } - free(pArray->pData); - free(pArray); + return NULL; } void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) { @@ -210,18 +211,18 @@ void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) { taosArrayDestroy(pArray); } -void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) { +void taosArraySort(SArray* pArray, __compar_fn_t compar) { assert(pArray != NULL); assert(compar != NULL); qsort(pArray->pData, pArray->size, pArray->elemSize, compar); } -void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn) { +void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { assert(pArray != NULL && comparFn != NULL); assert(key != NULL); - return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn); + return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); } void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { @@ -229,11 +230,11 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn); } -char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn) { +char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags) { assert(pArray != NULL); assert(key != NULL); - void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn); + void* p = taosbsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); if (p == NULL) { return NULL; } diff --git a/src/util/src/tbuffer.c b/src/util/src/tbuffer.c index 240f744ea3083e1b21f60076032483f828298167..a2cb32c1f45b1930036e545a5c1508cff658e0d9 100644 --- a/src/util/src/tbuffer.c +++ b/src/util/src/tbuffer.c @@ -191,7 +191,8 @@ double tbufReadDouble(SBufferReader* buf) { // writer functions void tbufCloseWriter( SBufferWriter* buf ) { - (*buf->allocator)( buf->data, 0 ); + tfree(buf->data); +// (*buf->allocator)( buf->data, 0 ); // potential memory leak. buf->data = NULL; buf->pos = 0; buf->size = 0; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 3afdf41d05cf131922a24c6641d02b95acfe2a93..c0cc8ce33942d1e6784db0f7f4b86757f8f7f593 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -510,7 +510,9 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { } pCacheObj->deleting = 1; - pthread_join(pCacheObj->refreshWorker, NULL); + if (taosCheckPthreadValid(pCacheObj->refreshWorker)) { + pthread_join(pCacheObj->refreshWorker, NULL); + } uInfo("cache:%s will be cleaned up", pCacheObj->name); doCleanupDataCache(pCacheObj); diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 75ac9307234cbf93c157ec091893dc3a6134ed8f..4d18ef14e2a1985259b11ff65391616c9d4706b2 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,4 +1,4 @@ -#include "taosdef.h" +#include "ttype.h" #include "tcompare.h" #include "tarray.h" @@ -277,7 +277,7 @@ int32_t taosArrayCompareString(const void* a, const void* b) { static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) { const SArray* arr = (const SArray*) pRight; - return taosArraySearchString(arr, pLeft, taosArrayCompareString) == NULL ? 0 : 1; + return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1; } static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { @@ -297,32 +297,14 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; switch (type) { - case TSDB_DATA_TYPE_SMALLINT: { - comparFn = compareInt16Val; break; - } - - case TSDB_DATA_TYPE_INT: { - comparFn = compareInt32Val; break; - } - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: { - comparFn = compareInt64Val; break; - } - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT:{ - comparFn = compareInt8Val; break; - } - - case TSDB_DATA_TYPE_FLOAT: { - comparFn = compareFloatVal; break; - } - - case TSDB_DATA_TYPE_DOUBLE: { - comparFn = compareDoubleVal; break; - } - + case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; + case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; + case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; + case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; + case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; case TSDB_DATA_TYPE_BINARY: { if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ comparFn = compareStrPatternComp; @@ -341,10 +323,14 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } else { comparFn = compareLenPrefixedWStr; } - break; } - + + case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; + case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; + case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; + case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; + default: comparFn = compareInt32Val; break; @@ -406,8 +392,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { switch (type) { case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: DEFAULT_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: DEFAULT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); + case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); + case TSDB_DATA_TYPE_FLOAT: DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); case TSDB_DATA_TYPE_TINYINT: diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 0a9f5a98c06c80efc123c6a05f4811b6c66c2615..c4bd57760222ac3da7d25510cc2f434fe0cf0cac 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -20,7 +20,7 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" -#include "tsystem.h" +#include "tsocket.h" #include "tutil.h" SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; @@ -95,6 +95,23 @@ static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { } } +static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) { + int32_t value = atoi(input_value); + uint16_t *option = (uint16_t *)cfg->ptr; + if (value < cfg->minValue || value > cfg->maxValue) { + uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", + cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); + } else { + if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { + *option = (uint16_t)value; + cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; + } else { + uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, + tsCfgStatusStr[cfg->cfgStatus], *option); + } + } +} + static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { int32_t value = atoi(input_value); int8_t *option = (int8_t *)cfg->ptr; @@ -112,32 +129,39 @@ static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { } } -static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { +static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { int length = (int)strlen(input_value); char *option = (char *)cfg->ptr; if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", - cfg->option, input_value, cfg->ptrLength, option); + uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option, + input_value, cfg->ptrLength, option); + return false; } else { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { wordexp_t full_path; if (0 != wordexp(input_value, &full_path, 0)) { printf("\nconfig dir: %s wordexp fail! reason:%s\n", input_value, strerror(errno)); wordfree(&full_path); - return; + return false; } - + if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) { strcpy(option, full_path.we_wordv[0]); } - + wordfree(&full_path); + char tmp[1025] = {0}; + if (realpath(option, tmp) != NULL) { + strcpy(option, tmp); + } + int code = taosMkDir(option, 0755); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - uError("config option:%s, input value:%s, directory not exist, create fail:%s", - cfg->option, input_value, strerror(errno)); + uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, + strerror(errno)); + return false; } cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; } else { @@ -145,6 +169,8 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { tsCfgStatusStr[cfg->cfgStatus], option); } } + + return true; } static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) { @@ -214,7 +240,7 @@ SGlobalCfg *taosGetConfigOption(const char *option) { return NULL; } -static void taosReadConfigOption(const char *option, char *value) { +static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) { for (int i = 0; i < tsGlobalConfigNum; ++i) { SGlobalCfg *cfg = tsGlobalConfig + i; if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue; @@ -230,6 +256,9 @@ static void taosReadConfigOption(const char *option, char *value) { case TAOS_CFG_VTYPE_INT32: taosReadInt32Config(cfg, value); break; + case TAOS_CFG_VTYPE_UINT16: + taosReadUInt16Config(cfg, value); + break; case TAOS_CFG_VTYPE_FLOAT: taosReadFloatConfig(cfg, value); break; @@ -242,6 +271,11 @@ static void taosReadConfigOption(const char *option, char *value) { case TAOS_CFG_VTYPE_DIRECTORY: taosReadDirectoryConfig(cfg, value); break; + case TAOS_CFG_VTYPE_DATA_DIRCTORY: + if (taosReadDirectoryConfig(cfg, value)) { + taosReadDataDirCfg(value, value2, value3); + } + break; default: uError("config option:%s, input value:%s, can't be recognized", option, value); break; @@ -322,8 +356,8 @@ void taosReadGlobalLogCfg() { } bool taosReadGlobalCfg() { - char * line, *option, *value, *value1; - int olen, vlen, vlen1; + char * line, *option, *value, *value2, *value3; + int olen, vlen, vlen2, vlen3; char fileName[PATH_MAX] = {0}; sprintf(fileName, "%s/taos.cfg", configDir); @@ -346,8 +380,8 @@ bool taosReadGlobalCfg() { while (!feof(fp)) { memset(line, 0, len); - option = value = NULL; - olen = vlen = 0; + option = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; tgetline(&line, &len, fp); line[len - 1] = 0; @@ -360,11 +394,14 @@ bool taosReadGlobalCfg() { if (vlen == 0) continue; value[vlen] = 0; - // For dataDir, the format is: - // dataDir /mnt/disk1 0 - paGetToken(value + vlen + 1, &value1, &vlen1); - - taosReadConfigOption(option, value); + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + + taosReadConfigOption(option, value, value2, value3); } fclose(fp); @@ -405,6 +442,9 @@ void taosPrintGlobalCfg() { case TAOS_CFG_VTYPE_INT32: uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; + case TAOS_CFG_VTYPE_UINT16: + uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_FLOAT: uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; @@ -419,6 +459,8 @@ void taosPrintGlobalCfg() { } taosPrintOsInfo(); + taosPrintDataDirCfg(); + uInfo("=================================="); } static void taosDumpCfg(SGlobalCfg *cfg) { @@ -440,6 +482,9 @@ static void taosDumpCfg(SGlobalCfg *cfg) { case TAOS_CFG_VTYPE_INT32: printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; + case TAOS_CFG_VTYPE_UINT16: + printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + break; case TAOS_CFG_VTYPE_FLOAT: printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 1eb6893cbc639be013104fe636f6af373a61b3c3..918ccc493543c785313df67c7f2a8eced6ba8fb7 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -24,17 +24,421 @@ typedef struct { int32_t val; const char* str; } STaosError; - #include "os.h" #include "taoserror.h" - static threadlocal int32_t tsErrno; int32_t* taosGetErrno() { return &tsErrno; } +#ifdef TAOS_ERROR_C +#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str=(msg)}, +#else +#define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = TAOS_DEF_ERROR_CODE(mod, code); +#endif + +#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) +#define TAOS_SUCCEEDED(err) ((err) >= 0) +#define TAOS_FAILED(err) ((err) < 0) + +#ifdef TAOS_ERROR_C +STaosError errors[] = { + {.val = 0, .str = "success"}, +#endif + +// rpc +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_REQUIRED, "Authentication required") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Redirect") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NOT_READY, "System not ready") // peer is not ready to process data +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_ALREADY_PROCESSED, "Message already processed") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED, "Last session not finished") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MISMATCHED_LINK_ID, "Mismatched meter id") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TOO_SLOW, "Processing of request timed out") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_MAX_SESSIONS, "Number of sessions reached limit") // too many sessions +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_APP_ERROR, "Unexpected generic error in RPC") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_UNEXPECTED_RESPONSE, "Unexpected response") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VALUE, "Invalid value") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID, "Invalid transaction id") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, "Invalid session id") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, "Invalid message type") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, "Invalid response type") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, "Client and server's time is not synchronized") +TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, "Database not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_VERSION, "Invalid app version") + +//common & util +TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, "Operation not supported") +TAOS_DEFINE_ERROR(TSDB_CODE_COM_MEMORY_CORRUPTED, "Memory corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_COM_OUT_OF_MEMORY, "Out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_COM_INVALID_CFG_MSG, "Invalid config message") +TAOS_DEFINE_ERROR(TSDB_CODE_COM_FILE_CORRUPTED, "Data file corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, "Invalid Ref ID") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_ALREADY_EXIST, "Ref is already there") +TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") + +//client +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_SQL, "Invalid SQL statement") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_QHANDLE, "Invalid qhandle") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TIME_STAMP, "Invalid combination of client/service time") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VALUE, "Invalid value in client") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_VERSION, "Invalid client version") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_IE, "Invalid client ie") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_FQDN, "Invalid host name") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_USER_LENGTH, "Invalid user name") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH, "Invalid password") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH, "Database name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH, "Table name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION, "Invalid connection") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY, "System out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_DISKSPACE, "System out of disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED, "Query cache erased") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED, "Query terminated") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY, "Result set too large to be sorted") // too many result for ordered super table projection query +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, "Application error") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, "Disconnected from service") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, "No write permission") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, "Connection killed") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, "Syntax error in SQL") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED, "Database not specified or available") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, "Table does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too long, check maxSQLLength config") + +// mnode +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Messag need to be reprocessed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Unexpected generic error in mnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_VERSION, "Incompatible protocol version") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_LEN, "Invalid message length") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_MSG_TYPE, "Invalid message type") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_SHELL_CONNS, "Too many connections") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_OUT_OF_MEMORY, "Out of memory in mnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, "Invalid stream id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, "Invalid connection id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_IS_RUNNING, "mnode is alreay running") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC, "failed to config sync") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_START_SYNC, "failed to start sync") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CREATE_DIR, "failed to create mnode dir") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_INIT_STEP, "failed to init components") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, "Object already there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, "Unexpected generic error in sdb") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE, "Invalid table type") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE, "Object not there") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW, "Invalid meta row") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE, "Invalid key type") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, "DNode already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, "DNode does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, "VGroup does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, "Master DNode cannot be removed") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, "Out of DNodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, "Cluster cfg inconsistent") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, "Invalid dnode cfg option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED, "Balance already enabled") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, "Vgroup not in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, "Vgroup already in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE, "Dnode not avaliable") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, "Cluster id not match") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, "Cluster not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED, "Dnode Id not configured") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED, "Dnode Ep not configured") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, "Account already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, "Invalid account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, "Invalid account options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, "Account authorization has expired") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, "User already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, "Invalid user") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER_FORMAT, "Invalid user format") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_PASS_FORMAT, "Invalid password format") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, "Can not get user from conn") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TABLE_ALREADY_EXIST, "Table already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_ID, "Table name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_NAME, "Table does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TABLE_TYPE, "Invalid table type in tsdb") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TAGS, "Too many tags") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_TIMESERIES, "Too many time series") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_SUPER_TABLE, "Not super table") // operation only available for super table +TAOS_DEFINE_ERROR(TSDB_CODE_MND_COL_NAME_TOO_LONG, "Tag name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_ALREAY_EXIST, "Tag already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_NOT_EXIST, "Tag does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST, "Field already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, "Cannot delete monitor database") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database not available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, "Database unsynced") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS, "Invalid database option: days out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP, "Invalid database option: keep >= keep1 >= keep0 >= days") + +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC, "Invalid topic name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_OPTION, "Invalid topic option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TOPIC_PARTITONS, "Invalid topic partitons num, valid range: [1, 1000]") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_ALREADY_EXIST, "Topic already exists") + +// dnode +TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, "Message not processed") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, "Dnode out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_NO_WRITE_ACCESS, "No permission for disk files in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_INVALID_MSG_LEN, "Invalid message length") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_DND_TOO_MANY_VNODES, "Too many vnode directories") + +// vnode +TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_IN_PROGRESS, "Action in progress") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_MSG_NOT_PROCESSED, "Message not processed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_ACTION_NEED_REPROCESSED, "Action need to be reprocessed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Invalid Vgroup ID") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INIT_FAILED, "Vnode initialization failed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISKSPACE, "System out of disk space") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS, "No write permission for disk files") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, "Missing data file") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, "Out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, "Unexpected generic error in vnode") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, "Invalid version file") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, "Database memory is full for commit failed") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FLOWCTRL, "Database memory is full for waiting commit") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_DROPPING, "Database is dropping") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_BALANCING, "Database is balancing") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, "Database suspended") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_SYNCING, "Database is syncing") + +// tsdb +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, "Invalid table type") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, "Invalid table schema version") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE, "No diskspace for tsdb") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS, "No permission for disk files") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, "Data file(s) corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, "Out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, "Tag too old") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, "Timestamp data out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, "Submit message is messed up") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, "Invalid operation") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, "Invalid creation of table") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, "No table data in memory skiplist") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, "File already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, "Need to reconfigure table") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, "Invalid information to create table") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "No available disk") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") + +// query +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, "Invalid message") // failed to validate the sql expression msg by vnode +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NO_DISKSPACE, "No diskspace for query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, "System out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, "Unexpected generic error in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, "Duplicated join key") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, "Query should response") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of this query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") + + +// grant +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, "DNode creation limited by licence") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_ACCT_LIMITED, "Account creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TIMESERIES_LIMITED, "Table creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DB_LIMITED, "DB creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_USER_LIMITED, "User creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CONN_LIMITED, "Conn creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STREAM_LIMITED, "Stream creation limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_SPEED_LIMITED, "Write speed limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STORAGE_LIMITED, "Storage capacity limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, "Query time limited by license") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, "CPU cores limited by license") + +// sync +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, "Invalid Sync Configuration") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, "Sync module not enabled") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, "Invalid Sync version") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, "Sync confirm expired") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TOO_MANY_FWDINFO, "Too many sync fwd infos") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_PROTOCOL, "Mismatched protocol") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_CLUSTERID, "Mismatched clusterId") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_MISMATCHED_SIGNATURE, "Mismatched signature") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CHECKSUM, "Invalid msg checksum") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGLEN, "Invalid msg length") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, "Invalid msg type") + +// wal +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Unexpected generic error in wal") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") + +// http +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, "http server is not onlin") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, "url is not support") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_URL, "invalid url format") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, "no enough memory") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUSET_TOO_BIG, "request size is too big") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_AUTH_INFO, "no auth info input") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_MSG_INPUT, "request is empty") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_SQL_INPUT, "no sql input") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_EXEC_USEDB, "no need to execute use db cmd") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SESSION_FULL, "session list was full") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR, "generate taosd token error") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_MULTI_REQUEST, "size of multi request is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, "failed to create gzip") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, "failed to finish gzip") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_LOGIN_FAILED, "failed to login") + +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, "invalid http version") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, "invalid content length") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, "invalid type of Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, "invalid format of Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, "invalid basic Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, "invalid taosd Authorization") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, "failed to parse method") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, "failed to parse target") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, "failed to parse http version") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, "failed to parse sp") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, "failed to parse status") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, "failed to parse phrase") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, "failed to parse crlf") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, "failed to parse header") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, "failed to parse header key") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, "failed to parse header val") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, "failed to parse chunk size") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, "failed to parse chunk") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, "failed to parse end section") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, "invalid parse state") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, "failed to parse error section") + +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_NULL, "query size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_SIZE, "query size can not more than 100") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR, "parse grafana json error") + +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_NOT_INPUT, "database name can not be null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_TOO_LONG, "database name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_INVALID_JSON, "invalid telegraf json fromat") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_NULL, "metrics size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_SIZE, "metrics size can not more than 1K") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NULL, "metric name not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_TYPE, "metric name type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_NULL, "metric name length is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_LONG, "metric name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_NULL, "timestamp not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE, "timestamp type should be integer") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL, "timestamp value smaller than 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_NULL, "tags not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, "tags size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, "tags size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NULL, "tag is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_NULL, "tag name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_SIZE, "tag name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE, "tag value type should be number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_NULL, "tag value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_NULL, "table is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_SIZE, "table name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_NULL, "fields not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_0, "fields size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG, "fields size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NULL, "field is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_NULL, "field name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE, "field name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE, "field value type should be number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL, "field value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_HOST_NOT_STRING, "host type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST, "stable not exist") + +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_NOT_INPUT, "database name can not be null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_TOO_LONG, "database name too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_INVALID_JSON, "invalid opentsdb json fromat") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_NULL, "metrics size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_SIZE, "metrics size can not more than 10K") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NULL, "metric name not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_TYPE, "metric name type should be string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_NULL, "metric name length is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_LONG, "metric name length can not more than 22") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_NULL, "timestamp not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE, "timestamp type should be integer") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL, "timestamp value smaller than 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_NULL, "tags not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_0, "tags size is 0") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG, "tags size too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NULL, "tag is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_NULL, "tag name is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_SIZE, "tag name length too long") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE, "tag value type should be boolean, number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_NULL, "tag value is null") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, "tag value can not more than 64") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, "value not find") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, "value type should be boolean, number or string") + +// odbc +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, "out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, "convertion not a valid literal input") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, "convertion undefined") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC_FRAC, "convertion fractional truncated") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_TRUNC, "convertion truncated") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_SUPPORT, "convertion not supported") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_OOR, "convertion numeric value out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OUT_OF_RANGE, "out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NOT_SUPPORT, "not supported yet") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_HANDLE, "invalid handle") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_RESULT, "no result set") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_NO_FIELDS, "no fields returned") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_INVALID_CURSOR, "invalid cursor") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_STATEMENT_NOT_READY, "statement not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONNECTION_BUSY, "connection still busy") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_CONNSTR, "bad connection string") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_BAD_ARG, "bad argument") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_NOT_VALID_TS, "not a valid timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_TOO_LARGE, "src too large") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, "src bad sequence") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, "src incomplete") +TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, "src general") + +// tfs +TAOS_DEFINE_ERROR(TSDB_CODE_FS_OUT_OF_MEMORY, "tfs out of memory") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_TOO_MANY_MOUNT, "tfs too many mount") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_DUP_PRIMARY, "tfs duplicate primary mount") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_PRIMARY_DISK, "tfs no primary mount") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_MOUNT_AT_TIER, "tfs no mount at tier") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, "tfs file already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") +TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") + +#ifdef TAOS_ERROR_C +}; +#endif static int tsCompareTaosError(const void* a, const void* b) { const STaosError* x = (const STaosError*)a; diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 64fea2843b6d54488c77f9da6aa4965749855229..dd621b3199189bc30f851c9a9289e4b10f6510f3 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -54,12 +54,12 @@ static int64_t tfOpenImp(int32_t fd) { } int64_t tfOpen(const char *pathname, int32_t flags) { - int32_t fd = open(pathname, flags); + int32_t fd = open(pathname, flags | O_BINARY); return tfOpenImp(fd); } int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = open(pathname, flags, mode); + int32_t fd = open(pathname, flags | O_BINARY, mode); return tfOpenImp(fd); } diff --git a/src/util/src/tidpool.c b/src/util/src/tidpool.c index 53d81bb5422e5c1b3b7384a97186a93940d26408..bd3f7745430a20bd0ba4250436e43e6524d1b431 100644 --- a/src/util/src/tidpool.c +++ b/src/util/src/tidpool.c @@ -104,10 +104,16 @@ void taosIdPoolCleanUp(void *handle) { int taosIdPoolNumOfUsed(void *handle) { id_pool_t *pIdPool = handle; - return pIdPool->maxId - pIdPool->numOfFree; + + pthread_mutex_lock(&pIdPool->mutex); + int ret = pIdPool->maxId - pIdPool->numOfFree; + pthread_mutex_unlock(&pIdPool->mutex); + + return ret; } -void taosIdPoolMarkStatus(void *handle, int id) { +bool taosIdPoolMarkStatus(void *handle, int id) { + bool ret = false; id_pool_t *pIdPool = handle; pthread_mutex_lock(&pIdPool->mutex); @@ -115,9 +121,13 @@ void taosIdPoolMarkStatus(void *handle, int id) { if (!pIdPool->freeList[slot]) { pIdPool->freeList[slot] = true; pIdPool->numOfFree--; + ret = true; + } else { + ret = false; } pthread_mutex_unlock(&pIdPool->mutex); + return ret; } int taosUpdateIdPool(id_pool_t *handle, int maxId) { @@ -147,6 +157,11 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { } int taosIdPoolMaxSize(void *handle) { - id_pool_t *pIdPool = (id_pool_t*)handle; - return pIdPool->maxId; + id_pool_t *pIdPool = (id_pool_t *)handle; + + pthread_mutex_lock(&pIdPool->mutex); + int ret = pIdPool->maxId; + pthread_mutex_unlock(&pIdPool->mutex); + + return ret; } \ No newline at end of file diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c deleted file mode 100644 index 2b1d13c78b3d52f511e0c01e5472a2cd19c5255b..0000000000000000000000000000000000000000 --- a/src/util/src/tkvstore.c +++ /dev/null @@ -1,621 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#define TAOS_RANDOM_FILE_FAIL_TEST -#include "os.h" -#include "hash.h" -#include "taoserror.h" -#include "tchecksum.h" -#include "tcoding.h" -#include "tkvstore.h" -#include "tulog.h" - -#define TD_KVSTORE_HEADER_SIZE 512 -#define TD_KVSTORE_MAJOR_VERSION 1 -#define TD_KVSTORE_MAINOR_VERSION 0 -#define TD_KVSTORE_SNAP_SUFFIX ".snap" -#define TD_KVSTORE_NEW_SUFFIX ".new" -#define TD_KVSTORE_INIT_MAGIC 0xFFFFFFFF - -typedef struct { - uint64_t uid; - int64_t offset; - int64_t size; -} SKVRecord; - -static int tdInitKVStoreHeader(int fd, char *fname); -static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo); -static void * tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo); -static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); -static char * tdGetKVStoreSnapshotFname(char *fdata); -static char * tdGetKVStoreNewFname(char *fdata); -static void tdFreeKVStore(SKVStore *pStore); -static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo); -static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t *version); -static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord); -static void * tdDecodeKVRecord(void *buf, SKVRecord *pRecord); -static int tdRestoreKVStore(SKVStore *pStore); - -int tdCreateKVStore(char *fname) { - int fd = open(fname, O_RDWR | O_CREAT, 0755); - if (fd < 0) { - uError("failed to open file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (tdInitKVStoreHeader(fd, fname) < 0) goto _err; - - if (fsync(fd) < 0) { - uError("failed to fsync file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (close(fd) < 0) { - uError("failed to close file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - return 0; - -_err: - if (fd >= 0) close(fd); - (void)remove(fname); - return -1; -} - -int tdDestroyKVStore(char *fname) { - if (remove(fname) < 0) { - uError("failed to remove file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) { - SStoreInfo info = {0}; - uint32_t version = 0; - - SKVStore *pStore = tdNewKVStore(fname, iFunc, aFunc, appH); - if (pStore == NULL) return NULL; - - pStore->fd = open(pStore->fname, O_RDWR); - if (pStore->fd < 0) { - uError("failed to open file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pStore->sfd = open(pStore->fsnap, O_RDONLY); - if (pStore->sfd < 0) { - if (errno != ENOENT) { - uError("failed to open file %s since %s", pStore->fsnap, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - } else { - uDebug("file %s exists, try to recover the KV store", pStore->fsnap); - if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info, &version) < 0) { - if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err; - } else { - if (version != KVSTORE_FILE_VERSION) { - uError("file %s version %u is not the same as program version %u, this may cause problem", pStore->fsnap, - version, KVSTORE_FILE_VERSION); - } - - if (taosFtruncate(pStore->fd, info.size) < 0) { - uError("failed to truncate %s to %" PRId64 " size since %s", pStore->fname, info.size, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err; - if (fsync(pStore->fd) < 0) { - uError("failed to fsync file %s since %s", pStore->fname, strerror(errno)); - goto _err; - } - } - - close(pStore->sfd); - pStore->sfd = -1; - (void)remove(pStore->fsnap); - } - - if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info, &version) < 0) goto _err; - if (version != KVSTORE_FILE_VERSION) { - uError("file %s version %u is not the same as program version %u, this may cause problem", pStore->fname, version, - KVSTORE_FILE_VERSION); - } - - pStore->info.size = TD_KVSTORE_HEADER_SIZE; - pStore->info.magic = info.magic; - - if (tdRestoreKVStore(pStore) < 0) goto _err; - - close(pStore->fd); - pStore->fd = -1; - - return pStore; - -_err: - if (pStore->fd > 0) { - close(pStore->fd); - pStore->fd = -1; - } - if (pStore->sfd > 0) { - close(pStore->sfd); - pStore->sfd = -1; - } - tdFreeKVStore(pStore); - return NULL; -} - -void tdCloseKVStore(SKVStore *pStore) { tdFreeKVStore(pStore); } - -int tdKVStoreStartCommit(SKVStore *pStore) { - ASSERT(pStore->fd < 0); - - pStore->fd = open(pStore->fname, O_RDWR); - if (pStore->fd < 0) { - uError("failed to open file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT, 0755); - if (pStore->sfd < 0) { - uError("failed to open file %s since %s", pStore->fsnap, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { - uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (fsync(pStore->sfd) < 0) { - uError("failed to fsync file %s since %s", pStore->fsnap, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (close(pStore->sfd) < 0) { - uError("failed to close file %s since %s", pStore->fsnap, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - pStore->sfd = -1; - - if (lseek(pStore->fd, 0, SEEK_END) < 0) { - uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(pStore->info.size == lseek(pStore->fd, 0, SEEK_CUR)); - - return 0; - -_err: - if (pStore->sfd > 0) { - close(pStore->sfd); - pStore->sfd = -1; - (void)remove(pStore->fsnap); - } - if (pStore->fd > 0) { - close(pStore->fd); - pStore->fd = -1; - } - return -1; -} - -int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen) { - SKVRecord rInfo = {0}; - char buf[64] = "\0"; - char * pBuf = buf; - - rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR); - if (rInfo.offset < 0) { - uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - rInfo.uid = uid; - rInfo.size = contLen; - - int tlen = tdEncodeKVRecord((void *)(&pBuf), &rInfo); - ASSERT(tlen == POINTER_DISTANCE(pBuf, buf)); - ASSERT(tlen == sizeof(SKVRecord)); - - if (taosWrite(pStore->fd, buf, tlen) < tlen) { - uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosWrite(pStore->fd, cont, contLen) < contLen) { - uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pStore->info.magic = - taosCalcChecksum(pStore->info.magic, (uint8_t *)POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); - pStore->info.size += (sizeof(SKVRecord) + contLen); - SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid)); - if (pRecord != NULL) { // just to insert - pStore->info.tombSize += pRecord->size; - } else { - pStore->info.nRecords++; - } - - taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); - uTrace("put uid %" PRIu64 " into kvStore %s", uid, pStore->fname); - - return 0; -} - -int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { - SKVRecord rInfo = {0}; - char buf[128] = "\0"; - - SKVRecord *pRecord = taosHashGet(pStore->map, (void *)(&uid), sizeof(uid)); - if (pRecord == NULL) { - uError("failed to drop KV store record with key %" PRIu64 " since not find", uid); - return -1; - } - - rInfo.offset = -pRecord->offset; - rInfo.uid = pRecord->uid; - rInfo.size = pRecord->size; - - void *pBuf = buf; - tdEncodeKVRecord(&pBuf, &rInfo); - - if (taosWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) { - uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pStore->info.magic = taosCalcChecksum(pStore->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf)); - pStore->info.size += POINTER_DISTANCE(pBuf, buf); - pStore->info.nDels++; - pStore->info.nRecords--; - pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); - - taosHashRemove(pStore->map, (void *)(&uid), sizeof(uid)); - uDebug("drop uid %" PRIu64 " from KV store %s", uid, pStore->fname); - - return 0; -} - -int tdKVStoreEndCommit(SKVStore *pStore) { - ASSERT(pStore->fd > 0); - - if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &(pStore->info)) < 0) return -1; - - if (fsync(pStore->fd) < 0) { - uError("failed to fsync file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (close(pStore->fd) < 0) { - uError("failed to close file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pStore->fd = -1; - - (void)remove(pStore->fsnap); - return 0; -} - -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { - char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; - SStoreInfo info = {0}; - - int fd = open(fname, O_RDONLY); - if (fd < 0) goto _err; - - if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err; - if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) goto _err; - - void *pBuf = (void *)buf; - pBuf = tdDecodeStoreInfo(pBuf, &info); - off_t offset = lseek(fd, 0, SEEK_END); - if (offset < 0) goto _err; - close(fd); - - *magic = info.magic; - *size = offset; - - return; - -_err: - if (fd >= 0) close(fd); - *magic = TD_KVSTORE_INIT_MAGIC; - *size = 0; -} - -static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t *version) { - char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; - - if (lseek(fd, 0, SEEK_SET) < 0) { - uError("failed to lseek file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { - uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) { - uError("file %s is broken", fname); - terrno = TSDB_CODE_COM_FILE_CORRUPTED; - return -1; - } - - void *pBuf = (void *)buf; - pBuf = tdDecodeStoreInfo(pBuf, pInfo); - pBuf = taosDecodeFixedU32(pBuf, version); - - return 0; -} - -static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { - char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; - - if (lseek(fd, 0, SEEK_SET) < 0) { - uError("failed to lseek file %s since %s", fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - void *pBuf = buf; - tdEncodeStoreInfo(&pBuf, pInfo); - taosEncodeFixedU32(&pBuf, KVSTORE_FILE_VERSION); - ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE); - - taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE); - if (taosWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { - uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -static int tdInitKVStoreHeader(int fd, char *fname) { - SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0, TD_KVSTORE_INIT_MAGIC}; - - return tdUpdateKVStoreHeader(fd, fname, &info); -} - -static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) { - int tlen = 0; - tlen += taosEncodeVariantI64(buf, pInfo->size); - tlen += taosEncodeVariantI64(buf, pInfo->tombSize); - tlen += taosEncodeVariantI64(buf, pInfo->nRecords); - tlen += taosEncodeVariantI64(buf, pInfo->nDels); - tlen += taosEncodeFixedU32(buf, pInfo->magic); - - return tlen; -} - -static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) { - buf = taosDecodeVariantI64(buf, &(pInfo->size)); - buf = taosDecodeVariantI64(buf, &(pInfo->tombSize)); - buf = taosDecodeVariantI64(buf, &(pInfo->nRecords)); - buf = taosDecodeVariantI64(buf, &(pInfo->nDels)); - buf = taosDecodeFixedU32(buf, &(pInfo->magic)); - - return buf; -} - -static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) { - SKVStore *pStore = (SKVStore *)calloc(1, sizeof(SKVStore)); - if (pStore == NULL) goto _err; - - pStore->fname = strdup(fname); - if (pStore->fname == NULL) { - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - pStore->fsnap = tdGetKVStoreSnapshotFname(fname); - if (pStore->fsnap == NULL) { - goto _err; - } - - pStore->fnew = tdGetKVStoreNewFname(fname); - if (pStore->fnew == NULL) goto _err; - - pStore->fd = -1; - pStore->sfd = -1; - pStore->nfd = -1; - pStore->iFunc = iFunc; - pStore->aFunc = aFunc; - pStore->appH = appH; - pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - if (pStore->map == NULL) { - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - return pStore; - -_err: - tdFreeKVStore(pStore); - return NULL; -} - -static void tdFreeKVStore(SKVStore *pStore) { - if (pStore) { - tfree(pStore->fname); - tfree(pStore->fsnap); - tfree(pStore->fnew); - taosHashCleanup(pStore->map); - free(pStore); - } -} - -static char *tdGetKVStoreSnapshotFname(char *fdata) { - size_t size = strlen(fdata) + strlen(TD_KVSTORE_SNAP_SUFFIX) + 1; - char * fname = malloc(size); - if (fname == NULL) { - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - return NULL; - } - sprintf(fname, "%s%s", fdata, TD_KVSTORE_SNAP_SUFFIX); - return fname; -} - -static char *tdGetKVStoreNewFname(char *fdata) { - size_t size = strlen(fdata) + strlen(TD_KVSTORE_NEW_SUFFIX) + 1; - char * fname = malloc(size); - if (fname == NULL) { - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - return NULL; - } - sprintf(fname, "%s%s", fdata, TD_KVSTORE_NEW_SUFFIX); - return fname; -} - -static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord) { - int tlen = 0; - tlen += taosEncodeFixedU64(buf, pRecord->uid); - tlen += taosEncodeFixedI64(buf, pRecord->offset); - tlen += taosEncodeFixedI64(buf, pRecord->size); - - return tlen; -} - -static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) { - buf = taosDecodeFixedU64(buf, &(pRecord->uid)); - buf = taosDecodeFixedI64(buf, &(pRecord->offset)); - buf = taosDecodeFixedI64(buf, &(pRecord->size)); - - return buf; -} - -static int tdRestoreKVStore(SKVStore *pStore) { - char tbuf[128] = "\0"; - void * buf = NULL; - int64_t maxBufSize = 0; - SKVRecord rInfo = {0}; - SKVRecord *pRecord = NULL; - - ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR)); - ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE); - - while (true) { - int64_t tsize = taosRead(pStore->fd, tbuf, sizeof(SKVRecord)); - if (tsize == 0) break; - if (tsize < sizeof(SKVRecord)) { - uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, - pStore->info.size, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - char *pBuf = tdDecodeKVRecord(tbuf, &rInfo); - ASSERT(POINTER_DISTANCE(pBuf, tbuf) == sizeof(SKVRecord)); - ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true); - - if (rInfo.offset < 0) { - taosHashRemove(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid)); - pStore->info.size += sizeof(SKVRecord); - pStore->info.nRecords--; - pStore->info.nDels++; - pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); - } else { - ASSERT(rInfo.offset > 0 && rInfo.size > 0); - if (taosHashPut(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) { - uError("failed to put record in KV store %s", pStore->fname); - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - maxBufSize = MAX(maxBufSize, rInfo.size); - - if (lseek(pStore->fd, (off_t)rInfo.size, SEEK_CUR) < 0) { - uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pStore->info.size += (sizeof(SKVRecord) + rInfo.size); - pStore->info.nRecords++; - } - } - - buf = malloc((size_t)maxBufSize); - if (buf == NULL) { - uError("failed to allocate %" PRId64 " bytes in KV store %s", maxBufSize, pStore->fname); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pRecord = taosHashIterate(pStore->map, NULL); - while (pRecord) { - if (lseek(pStore->fd, (off_t)(pRecord->offset + sizeof(SKVRecord)), SEEK_SET) < 0) { - uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) { - uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname, - strerror(errno), pRecord->offset); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (pStore->iFunc) { - if ((*pStore->iFunc)(pStore->appH, buf, (int)pRecord->size) < 0) { - uError("failed to restore record uid %" PRIu64 " in kv store %s at offset %" PRId64 " size %" PRId64 - " since %s", - pRecord->uid, pStore->fname, pRecord->offset, pRecord->size, tstrerror(terrno)); - goto _err; - } - } - - pRecord = taosHashIterate(pStore->map, pRecord); - } - - if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); - - tfree(buf); - return 0; - -_err: - taosHashCancelIterate(pStore->map, pRecord); - tfree(buf); - return -1; -} diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index 8c2ad83de117aaf528565a711a2aa3732984e0a9..2f52551e2ac7c79ef54cd4546be7bf844ff04980 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -38,11 +38,13 @@ void tdListEmpty(SList *list) { list->numOfEles = 0; } -void tdListFree(SList *list) { +void *tdListFree(SList *list) { if (list) { tdListEmpty(list); free(list); } + + return NULL; } void tdListPrependNode(SList *list, SListNode *node) { diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index e0fe51e22a6405273cbb60fbfa04d3a5956105b0..448bb3ecd1bfa8ef17bd7c87a464c095e1093d6d 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -28,10 +28,13 @@ #define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) #define LOG_FILE_NAME_LEN 300 -#define TSDB_DEFAULT_LOG_BUF_SIZE (512 * 1024) // 512K -#define TSDB_MIN_LOG_BUF_SIZE 1024 // 1K -#define TSDB_MAX_LOG_BUF_SIZE (1024 * 1024) // 1M -#define TSDB_DEFAULT_LOG_BUF_UNIT 1024 // 1K +#define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB + +#define DEFAULT_LOG_INTERVAL 25 +#define LOG_INTERVAL_STEP 5 +#define MIN_LOG_INTERVAL 5 +#define MAX_LOG_INTERVAL 25 +#define LOG_MAX_WAIT_MSEC 1000 #define LOG_BUF_BUFFER(x) ((x)->buffer) #define LOG_BUF_START(x) ((x)->buffStart) @@ -44,6 +47,7 @@ typedef struct { int32_t buffStart; int32_t buffEnd; int32_t buffSize; + int32_t minBuffSize; int32_t fd; int32_t stop; pthread_t asyncThread; @@ -68,6 +72,15 @@ int8_t tsAsyncLog = 1; float tsTotalLogDirGB = 0; float tsAvailLogDirGB = 0; float tsMinimalLogDirGB = 1.0f; +int64_t asyncLogLostLines = 0; +int32_t writeInterval = DEFAULT_LOG_INTERVAL; + +int64_t dbgEmptyW = 0; +int64_t dbgWN = 0; +int64_t dbgSmallWN = 0; +int64_t dbgBigWN = 0; +int64_t dbgWSize = 0; + #ifdef _TD_POWER_ char tsLogDir[TSDB_FILENAME_LEN] = "/var/log/power"; #else @@ -108,7 +121,8 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); - tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); + //tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); + taosMsleep(MAX_LOG_INTERVAL/1000); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { pthread_join(tsLogObj.logHandle->asyncThread, NULL); } @@ -171,7 +185,9 @@ static void *taosThreadToOpenNewFile(void *param) { int32_t fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { - uError("open new log file fail! fd:%d reason:%s", fd, strerror(errno)); + tsLogObj.openInProgress = 0; + tsLogObj.lines = tsLogObj.maxLines - 1000; + uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno)); return NULL; } @@ -363,7 +379,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, + len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); len += sprintf(buffer + len, "%s", flags); @@ -449,7 +465,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, + len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); len += sprintf(buffer + len, "%s", flags); @@ -495,8 +511,6 @@ static void taosCloseLogByFd(int32_t fd) { static SLogBuff *taosLogBuffNew(int32_t bufSize) { SLogBuff *tLogBuff = NULL; - if (bufSize < TSDB_MIN_LOG_BUF_SIZE || bufSize > TSDB_MAX_LOG_BUF_SIZE) return NULL; - tLogBuff = calloc(1, sizeof(SLogBuff)); if (tLogBuff == NULL) return NULL; @@ -505,10 +519,11 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { LOG_BUF_START(tLogBuff) = LOG_BUF_END(tLogBuff) = 0; LOG_BUF_SIZE(tLogBuff) = bufSize; + tLogBuff->minBuffSize = bufSize / 10; tLogBuff->stop = 0; if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; - tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); + //tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); return tLogBuff; @@ -527,10 +542,27 @@ static void taosLogBuffDestroy(SLogBuff *tLogBuff) { } #endif +static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, char *msg, int32_t msgLen) { + if (start > end) { + memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); + } else { + if (LOG_BUF_SIZE(tLogBuff) - end < msgLen) { + memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, LOG_BUF_SIZE(tLogBuff) - end); + memcpy(LOG_BUF_BUFFER(tLogBuff), msg + LOG_BUF_SIZE(tLogBuff) - end, msgLen - LOG_BUF_SIZE(tLogBuff) + end); + } else { + memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); + } + } + LOG_BUF_END(tLogBuff) = (LOG_BUF_END(tLogBuff) + msgLen) % LOG_BUF_SIZE(tLogBuff); +} + static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) { int32_t start = 0; int32_t end = 0; int32_t remainSize = 0; + static int64_t lostLine = 0; + char tmpBuf[40] = {0}; + int32_t tmpBufLen = 0; if (tLogBuff == NULL || tLogBuff->stop) return -1; @@ -538,79 +570,128 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) start = LOG_BUF_START(tLogBuff); end = LOG_BUF_END(tLogBuff); - remainSize = (start > end) ? (end - start - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1); + remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1); + + if (lostLine > 0) { + sprintf(tmpBuf, "...Lost %"PRId64" lines here...\n", lostLine); + tmpBufLen = (int32_t)strlen(tmpBuf); + } - if (remainSize <= msgLen) { + if (remainSize <= msgLen || ((lostLine > 0) && (remainSize <= (msgLen + tmpBufLen)))) { + lostLine++; + asyncLogLostLines++; pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); return -1; } - if (start > end) { - memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); - } else { - if (LOG_BUF_SIZE(tLogBuff) - end < msgLen) { - memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, LOG_BUF_SIZE(tLogBuff) - end); - memcpy(LOG_BUF_BUFFER(tLogBuff), msg + LOG_BUF_SIZE(tLogBuff) - end, msgLen - LOG_BUF_SIZE(tLogBuff) + end); - } else { - memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); - } + if (lostLine > 0) { + taosCopyLogBuffer(tLogBuff, start, end, tmpBuf, tmpBufLen); + lostLine = 0; } - LOG_BUF_END(tLogBuff) = (LOG_BUF_END(tLogBuff) + msgLen) % LOG_BUF_SIZE(tLogBuff); - // TODO : put string in the buffer + taosCopyLogBuffer(tLogBuff, LOG_BUF_START(tLogBuff), LOG_BUF_END(tLogBuff), msg, msgLen); - tsem_post(&(tLogBuff->buffNotEmpty)); + //int32_t w = atomic_sub_fetch_32(&waitLock, 1); + /* + if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(tLogBuff) * 4 /5))) { + tsem_post(&(tLogBuff->buffNotEmpty)); + dbgPostN++; + } else { + dbgNoPostN++; + } + */ pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); + return 0; } -static int32_t taosPollLogBuffer(SLogBuff *tLogBuff, char *buf, int32_t bufSize) { - int32_t start = LOG_BUF_START(tLogBuff); - int32_t end = LOG_BUF_END(tLogBuff); - int32_t pollSize = 0; +static int32_t taosGetLogRemainSize(SLogBuff *tLogBuff, int32_t start, int32_t end) { + int32_t rSize = end - start; - if (start == end) { - return 0; - } else if (start < end) { - pollSize = MIN(end - start, bufSize); + return rSize >= 0 ? rSize : LOG_BUF_SIZE(tLogBuff) + rSize; +} - memcpy(buf, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); - return pollSize; - } else { - pollSize = MIN(end + LOG_BUF_SIZE(tLogBuff) - start, bufSize); - if (pollSize > LOG_BUF_SIZE(tLogBuff) - start) { - int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - memcpy(buf, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - memcpy(buf + tsize, LOG_BUF_BUFFER(tLogBuff), pollSize - tsize); +static void taosWriteLog(SLogBuff *tLogBuff) { + static int32_t lastDuration = 0; + int32_t remainChecked = 0; + int32_t start, end, pollSize; + + do { + if (remainChecked == 0) { + start = LOG_BUF_START(tLogBuff); + end = LOG_BUF_END(tLogBuff); + + if (start == end) { + dbgEmptyW++; + writeInterval = MAX_LOG_INTERVAL; + return; + } + pollSize = taosGetLogRemainSize(tLogBuff, start, end); + if (pollSize < tLogBuff->minBuffSize) { + lastDuration += writeInterval; + if (lastDuration < LOG_MAX_WAIT_MSEC) { + break; + } + } + + lastDuration = 0; + } + + if (start < end) { + taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { - memcpy(buf, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); + int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; + taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + + taosWrite(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); } - return pollSize; - } + + dbgWN++; + dbgWSize+=pollSize; + + if (pollSize < tLogBuff->minBuffSize) { + dbgSmallWN++; + if (writeInterval < MAX_LOG_INTERVAL) { + writeInterval += LOG_INTERVAL_STEP; + } + } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/3) { + dbgBigWN++; + writeInterval = MIN_LOG_INTERVAL; + } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/4) { + if (writeInterval > MIN_LOG_INTERVAL) { + writeInterval -= LOG_INTERVAL_STEP; + } + } + + LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + pollSize) % LOG_BUF_SIZE(tLogBuff); + + start = LOG_BUF_START(tLogBuff); + end = LOG_BUF_END(tLogBuff); + + pollSize = taosGetLogRemainSize(tLogBuff, start, end); + if (pollSize < tLogBuff->minBuffSize) { + break; + } + + writeInterval = MIN_LOG_INTERVAL; + + remainChecked = 1; + }while (1); } static void *taosAsyncOutputLog(void *param) { SLogBuff *tLogBuff = (SLogBuff *)param; - int32_t log_size = 0; - - char tempBuffer[TSDB_DEFAULT_LOG_BUF_UNIT]; - + while (1) { - tsem_wait(&(tLogBuff->buffNotEmpty)); + //tsem_wait(&(tLogBuff->buffNotEmpty)); + + taosMsleep(writeInterval); // Polling the buffer - while (1) { - log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT); - if (log_size) { - taosWrite(tLogBuff->fd, tempBuffer, log_size); - LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff); - } else { - break; - } - } + taosWriteLog(tLogBuff); if (tLogBuff->stop) break; } diff --git a/src/util/src/tnettest.c b/src/util/src/tnettest.c index e1b834b9499311d6932e5778a0e8116756a11961..28abad356c156182b23feebd8d2e80ea4c914e52 100644 --- a/src/util/src/tnettest.c +++ b/src/util/src/tnettest.c @@ -23,6 +23,8 @@ #include "tsocket.h" #include "trpc.h" #include "rpcHead.h" +#include "tchecksum.h" +#include "syncMsg.h" #define MAX_PKG_LEN (64 * 1000) #define BUFFER_SIZE (MAX_PKG_LEN + 1024) @@ -48,7 +50,7 @@ static void *taosNetBindUdpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { + if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { uError("failed to create UDP socket since %s", strerror(errno)); return NULL; } @@ -104,7 +106,7 @@ static void *taosNetBindTcpPort(void *sarg) { struct sockaddr_in server_addr; struct sockaddr_in clientAddr; - STestInfo *pinfo = sarg; + STestInfo *pinfo = sarg; int32_t port = pinfo->port; SOCKET serverSocket; int32_t addr_len = sizeof(clientAddr); @@ -133,7 +135,6 @@ static void *taosNetBindTcpPort(void *sarg) { return NULL; } - if (taosKeepTcpAlive(serverSocket) < 0) { uError("failed to set tcp server keep-alive option since %s", strerror(errno)); taosCloseSocket(serverSocket); @@ -178,8 +179,8 @@ static void *taosNetBindTcpPort(void *sarg) { } static int32_t taosNetCheckTcpPort(STestInfo *info) { - SOCKET clientSocket; - char buffer[BUFFER_SIZE] = {0}; + SOCKET clientSocket; + char buffer[BUFFER_SIZE] = {0}; if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { uError("failed to create TCP client socket since %s", strerror(errno)); @@ -409,13 +410,51 @@ static void taosNetTestStartup(char *host, int32_t port) { free(pStep); } +static void taosNetCheckSync(char *host, int32_t port) { + uint32_t ip = taosGetIpv4FromFqdn(host); + if (ip == 0xffffffff) { + uError("failed to get IP address from %s since %s", host, strerror(errno)); + return; + } + + SOCKET connFd = taosOpenTcpClientSocket(ip, (uint16_t)port, 0); + if (connFd < 0) { + uError("failed to create socket while test port:%d since %s", port, strerror(errno)); + return; + } + + SSyncMsg msg; + memset(&msg, 0, sizeof(SSyncMsg)); + SSyncHead *pHead = &msg.head; + pHead->type = TAOS_SMSG_TEST; + pHead->protocol = SYNC_PROTOCOL_VERSION; + pHead->signature = SYNC_SIGNATURE; + pHead->code = 0; + pHead->cId = 0; + pHead->vgId = -1; + pHead->len = sizeof(SSyncMsg) - sizeof(SSyncHead); + taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SSyncHead)); + + if (taosWriteMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + uError("failed to test port:%d while send msg since %s", port, strerror(errno)); + return; + } + + if (taosReadMsg(connFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) { + uError("failed to test port:%d while recv msg since %s", port, strerror(errno)); + } + + uInfo("successed to test TCP port:%d", port); + taosCloseSocket(connFd); +} + static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { - int32_t endPort = startPort + 9; + int32_t endPort = startPort + TSDB_PORT_SYNC; char spi = 0; uInfo("check rpc, host:%s startPort:%d endPort:%d pkgLen:%d\n", host, startPort, endPort, pkgLen); - - for (uint16_t port = startPort; port <= endPort; port++) { + + for (uint16_t port = startPort; port < endPort; port++) { int32_t sendpkgLen; if (pkgLen <= tsRpcMaxUdpSize) { sendpkgLen = tsRpcMaxUdpSize + 1000; @@ -443,6 +482,9 @@ static void taosNetTestRpc(char *host, int32_t startPort, int32_t pkgLen) { uInfo("successed to test UDP port:%d", port); } } + + taosNetCheckSync(host, startPort + TSDB_PORT_SYNC); + taosNetCheckSync(host, startPort + TSDB_PORT_ARBITRATOR); } static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { @@ -509,6 +551,8 @@ void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen) { taosNetTestServer(host, port, pkgLen); } else if (0 == strcmp("rpc", role)) { taosNetTestRpc(host, port, pkgLen); + } else if (0 == strcmp("sync", role)) { + taosNetCheckSync(host, port); } else if (0 == strcmp("startup", role)) { taosNetTestStartup(host, port); } else { diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 56f2322a7bd1e0778e2d52667663b3baf5c08c1f..c6d49dfb3d8242bdf697b3212fde479f01f8bcd9 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -248,7 +248,7 @@ void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { gettimeofday(&timeSecs, NULL); curTime = timeSecs.tv_sec; ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d 0x%08" PRIx64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, + len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); va_start(argpointer, format); len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 3ef45e9b191fddc45220c7b07551b098ae0891ba..7d64bd1f83fb8d235c825057251a5e76e0b96b2a 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -24,22 +24,22 @@ #define TSDB_REF_STATE_DELETED 2 typedef struct SRefNode { - struct SRefNode *prev; // previous node + struct SRefNode *prev; // previous node struct SRefNode *next; // next node - void *p; // pointer to resource protected, + void *p; // pointer to resource protected, int64_t rid; // reference ID int32_t count; // number of references - int removed; // 1: removed + int removed; // 1: removed } SRefNode; typedef struct { SRefNode **nodeList; // array of SRefNode linked list int state; // 0: empty, 1: active; 2: deleted - int rsetId; // refSet ID, global unique + int rsetId; // refSet ID, global unique int64_t rid; // increase by one for each new reference - int max; // mod + int max; // mod int32_t count; // total number of SRefNodes in this set - int64_t *lockedBy; + int64_t *lockedBy; void (*fp)(void *); } SRefSet; @@ -62,9 +62,9 @@ int taosOpenRef(int max, void (*fp)(void *)) SRefSet *pSet; int64_t *lockedBy; int i, rsetId; - + pthread_once(&tsRefModuleInit, taosInitRefModule); - + nodeList = calloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; @@ -79,12 +79,12 @@ int taosOpenRef(int max, void (*fp)(void *)) } pthread_mutex_lock(&tsRefMutex); - + for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; - } + } if (i < TSDB_REF_OBJECTS) { rsetId = tsNextId; @@ -105,7 +105,7 @@ int taosOpenRef(int max, void (*fp)(void *)) free (nodeList); free (lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); - } + } pthread_mutex_unlock(&tsRefMutex); @@ -127,7 +127,7 @@ int taosCloseRef(int rsetId) pthread_mutex_lock(&tsRefMutex); - if (pSet->state == TSDB_REF_STATE_ACTIVE) { + if (pSet->state == TSDB_REF_STATE_ACTIVE) { pSet->state = TSDB_REF_STATE_DELETED; deleted = 1; uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count); @@ -142,7 +142,7 @@ int taosCloseRef(int rsetId) return 0; } -int64_t taosAddRef(int rsetId, void *p) +int64_t taosAddRef(int rsetId, void *p) { int hash; SRefNode *pNode; @@ -163,7 +163,7 @@ int64_t taosAddRef(int rsetId, void *p) terrno = TSDB_CODE_REF_ID_REMOVED; return -1; } - + pNode = calloc(sizeof(SRefNode), 1); if (pNode == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; @@ -173,7 +173,7 @@ int64_t taosAddRef(int rsetId, void *p) rid = atomic_add_fetch_64(&pSet->rid, 1); hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); - + pNode->p = p; pNode->rid = rid; pNode->count = 1; @@ -187,16 +187,16 @@ int64_t taosAddRef(int rsetId, void *p) taosUnlockList(pSet->lockedBy+hash); - return rid; + return rid; } -int taosRemoveRef(int rsetId, int64_t rid) +int taosRemoveRef(int rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid -void *taosAcquireRef(int rsetId, int64_t rid) +void *taosAcquireRef(int rsetId, int64_t rid) { int hash; SRefNode *pNode; @@ -204,7 +204,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) void *p = NULL; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { - uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); + //uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); terrno = TSDB_CODE_REF_INVALID_ID; return NULL; } @@ -223,7 +223,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) terrno = TSDB_CODE_REF_ID_REMOVED; return NULL; } - + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); @@ -233,8 +233,8 @@ void *taosAcquireRef(int rsetId, int64_t rid) if (pNode->rid == rid) { break; } - - pNode = pNode->next; + + pNode = pNode->next; } if (pNode) { @@ -258,7 +258,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) return p; } -int taosReleaseRef(int rsetId, int64_t rid) +int taosReleaseRef(int rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0); } @@ -274,12 +274,13 @@ void *taosIterateRef(int rsetId, int64_t rid) { return NULL; } - if (rid <= 0) { + if (rid < 0) { uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; return NULL; } + void *newP = NULL; pSet = tsRefSetList + rsetId; taosIncRsetCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { @@ -289,52 +290,68 @@ void *taosIterateRef(int rsetId, int64_t rid) { return NULL; } - int hash = 0; - if (rid > 0) { - hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + do { + newP = NULL; + int hash = 0; + if (rid > 0) { + hash = rid % pSet->max; + taosLockList(pSet->lockedBy+hash); - pNode = pSet->nodeList[hash]; - while (pNode) { - if (pNode->rid == rid) break; - pNode = pNode->next; - } + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->rid == rid) break; + pNode = pNode->next; + } - if (pNode == NULL) { - uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); - terrno = TSDB_CODE_REF_NOT_EXIST; - taosUnlockList(pSet->lockedBy+hash); - return NULL; + if (pNode == NULL) { + uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + taosUnlockList(pSet->lockedBy+hash); + taosDecRsetCount(pSet); + return NULL; + } + + // rid is there + pNode = pNode->next; + // check first place + while (pNode) { + if (!pNode->removed) break; + pNode = pNode->next; + } + if (pNode == NULL) { + taosUnlockList(pSet->lockedBy+hash); + hash++; + } } - // rid is there - pNode = pNode->next; - if (pNode == NULL) { - taosUnlockList(pSet->lockedBy+hash); - hash++; + if (pNode == NULL) { + for (; hash < pSet->max; ++hash) { + taosLockList(pSet->lockedBy+hash); + pNode = pSet->nodeList[hash]; + if (pNode) { + // check first place + while (pNode) { + if (!pNode->removed) break; + pNode = pNode->next; + } + if (pNode) break; + } + taosUnlockList(pSet->lockedBy+hash); + } } - } - if (pNode == NULL) { - for (; hash < pSet->max; ++hash) { - taosLockList(pSet->lockedBy+hash); - pNode = pSet->nodeList[hash]; - if (pNode) break; + if (pNode) { + pNode->count++; // acquire it + newP = pNode->p; taosUnlockList(pSet->lockedBy+hash); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); + } else { + uTrace("rsetId:%d the list is over", rsetId); } - } - - void *newP = NULL; - if (pNode) { - pNode->count++; // acquire it - newP = pNode->p; - taosUnlockList(pSet->lockedBy+hash); - uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); - } else { - uTrace("rsetId:%d the list is over", rsetId); - } - if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one + if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one + if (pNode) rid = pNode->rid; + } while (newP && pNode->removed); taosDecRsetCount(pSet); @@ -350,22 +367,22 @@ int taosListRef() { for (int i = 0; i < TSDB_REF_OBJECTS; ++i) { pSet = tsRefSetList + i; - - if (pSet->state == TSDB_REF_STATE_EMPTY) + + if (pSet->state == TSDB_REF_STATE_EMPTY) continue; uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count); for (int j=0; j < pSet->max; ++j) { pNode = pSet->nodeList[j]; - + while (pNode) { uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count); pNode = pNode->next; num++; } - } - } + } + } pthread_mutex_unlock(&tsRefMutex); @@ -397,16 +414,16 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { terrno = TSDB_CODE_REF_ID_REMOVED; return -1; } - + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); - + pNode = pSet->nodeList[hash]; while (pNode) { if (pNode->rid == rid) break; - pNode = pNode->next; + pNode = pNode->next; } if (pNode) { @@ -417,18 +434,18 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (pNode->prev) { pNode->prev->next = pNode->next; } else { - pSet->nodeList[hash] = pNode->next; + pSet->nodeList[hash] = pNode->next; } if (pNode->next) { - pNode->next->prev = pNode->prev; - } + pNode->next->prev = pNode->prev; + } released = 1; } else { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); } } else { - uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; code = -1; } @@ -437,11 +454,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (released) { uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); - (*pSet->fp)(pNode->p); + (*pSet->fp)(pNode->p); free(pNode); taosDecRsetCount(pSet); - } + } return code; } @@ -493,5 +510,5 @@ static void taosDecRsetCount(SRefSet *pSet) { } pthread_mutex_unlock(&tsRefMutex); -} +} diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 49b69ea0a13de346136b6196e4535afe3b7f337b..77941cba82010a9187227b4740c4100680577403 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -22,6 +22,8 @@ #define SIGPIPE EPIPE #endif +#define TCP_CONN_TIMEOUT 3000 // conn timeout + int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; @@ -32,14 +34,27 @@ int32_t taosGetFqdn(char *fqdn) { struct addrinfo hints = {0}; struct addrinfo *result = NULL; +#ifdef __APPLE__ + // on macosx, hostname -f has the form of xxx.local + // which will block getaddrinfo for a few seconds if AI_CANONNAME is set + // thus, we choose AF_INET (ipv4 for the moment) to make getaddrinfo return + // immediately + hints.ai_family = AF_INET; +#else // __APPLE__ hints.ai_flags = AI_CANONNAME; +#endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } +#ifdef __APPLE__ + // refer to comments above + strcpy(fqdn, hostname); +#else // __APPLE__ strcpy(fqdn, result->ai_canonname); +#endif // __APPLE__ freeaddrinfo(result); return 0; } @@ -241,7 +256,7 @@ int32_t taosReadn(SOCKET fd, char *ptr, int32_t nbytes) { SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; - SOCKET sockFd; + SOCKET sockFd; int32_t bufSize = 1024000; uDebug("open udp socket:0x%x:%hu", ip, port); @@ -251,7 +266,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = (int32_t)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { + if ((sockFd = socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -280,7 +295,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { } SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientIp) { - SOCKET sockFd = 0; + SOCKET sockFd = 0; int32_t ret; struct sockaddr_in serverAddr, clientAddr; int32_t bufSize = 1024 * 1024; @@ -333,10 +348,47 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie serverAddr.sin_addr.s_addr = destIp; serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); - ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); +#ifdef _TD_LINUX + taosSetNonblocking(sockFd, 1); + ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); + if (ret == -1) { + if (errno == EHOSTUNREACH) { + uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) { + struct pollfd wfd[1]; + + wfd[0].fd = sockFd; + wfd[0].events = POLLOUT; + + int res = poll(wfd, 1, TCP_CONN_TIMEOUT); + if (res == -1 || res == 0) { + uError("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + int optVal = -1, optLen = sizeof(int); + if ((0 != taosGetSockOpt(sockFd, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { + uError("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + ret = 0; + } else { // Other error + uError("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + } + taosSetNonblocking(sockFd, 0); + +#else + ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); +#endif if (ret != 0) { - // uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); taosCloseSocket(sockFd); sockFd = -1; } else { @@ -354,6 +406,8 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { return -1; } +#ifndef __APPLE__ + // all fails on macosx int32_t probes = 3; if (taosSetSockOpt(sockFd, SOL_TCP, TCP_KEEPCNT, (void *)&probes, sizeof(probes)) < 0) { uError("fd:%d setsockopt SO_KEEPCNT failed: %d (%s)", sockFd, errno, strerror(errno)); @@ -374,6 +428,7 @@ int32_t taosKeepTcpAlive(SOCKET sockFd) { taosCloseSocket(sockFd); return -1; } +#endif // __APPLE__ int32_t nodelay = 1; if (taosSetSockOpt(sockFd, IPPROTO_TCP, TCP_NODELAY, (void *)&nodelay, sizeof(nodelay)) < 0) { @@ -406,7 +461,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = (int32_t)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + if ((sockFd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); taosCloseSocketNoCheck(sockFd); return -1; @@ -449,36 +504,36 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { #define COPY_SIZE 32768 // sendfile shall be used -int32_t taosCopyFds(SOCKET sfd, SOCKET dfd, int64_t len) { +int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) { int64_t leftLen; - int32_t readLen, writeLen; + int64_t readLen, writeLen; char temp[COPY_SIZE]; leftLen = len; while (leftLen > 0) { if (leftLen < COPY_SIZE) - readLen = (int32_t)leftLen; + readLen = leftLen; else readLen = COPY_SIZE; // 4K - int32_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen); + int64_t retLen = taosReadMsg(sfd, temp, (int32_t)readLen); if (readLen != retLen) { - uError("read error, readLen:%d retLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, retLen, len, - leftLen, strerror(errno)); + uError("read error, readLen:%" PRId64 " retLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", + readLen, retLen, len, leftLen, strerror(errno)); return -1; } - writeLen = taosWriteMsg(dfd, temp, readLen); + writeLen = taosWriteMsg(dfd, temp, (int32_t)readLen); if (readLen != writeLen) { - uError("copy error, readLen:%d writeLen:%d len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", readLen, writeLen, - len, leftLen, strerror(errno)); + uError("copy error, readLen:%" PRId64 " writeLen:%" PRId64 " len:%" PRId64 " leftLen:%" PRId64 ", reason:%s", + readLen, writeLen, len, leftLen, strerror(errno)); return -1; } leftLen -= readLen; } - return 0; + return len; } diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 015c687b3d941db3f6414d0d3d92564dd14e0fdb..98334497772d60991a553b5d929404ce5e3437e4 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -18,6 +18,7 @@ #include "tsched.h" #include "ttimer.h" #include "tutil.h" +#include "monotonic.h" extern int8_t tscEmbedded; @@ -186,6 +187,10 @@ static void removeTimer(uintptr_t id) { unlockTimerList(list); } +static int64_t getMonotonicMs(void) { + return (int64_t) getMonotonicUs() / 1000; +} + static void addToWheel(tmr_obj_t* timer, uint32_t delay) { timerAddRef(timer); // select a wheel for the timer, we are not an accurate timer, @@ -201,7 +206,7 @@ static void addToWheel(tmr_obj_t* timer, uint32_t delay) { time_wheel_t* wheel = wheels + timer->wheel; timer->prev = NULL; - timer->expireAt = taosGetTimestampMs() + delay; + timer->expireAt = getMonotonicMs() + delay; pthread_mutex_lock(&wheel->mutex); @@ -334,7 +339,7 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle } static void taosTimerLoopFunc(int signo) { - int64_t now = taosGetTimestampMs(); + int64_t now = getMonotonicMs(); for (int i = 0; i < tListLen(wheels); i++) { // `expried` is a temporary expire list. @@ -501,7 +506,7 @@ static void taosTmrModuleInit(void) { pthread_mutex_init(&tmrCtrlMutex, NULL); - int64_t now = taosGetTimestampMs(); + int64_t now = getMonotonicMs(); for (int i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { @@ -532,6 +537,9 @@ static void taosTmrModuleInit(void) { } void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) { + const char* ret = monotonicInit(); + tmrInfo("ttimer monotonic clock source:%s", ret); + pthread_once(&tmrModuleInit, taosTmrModuleInit); pthread_mutex_lock(&tmrCtrlMutex); diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 6066d58416e8a5ad905e3b0706f03adc013fa445..0c96ed2a2f3dfb7f03268c9f8fbb1b0afa2397b9 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) FIND_PATH(HEADER_GTEST_INCLUDE_DIR gtest.h /usr/include/gtest /usr/local/include/gtest) diff --git a/src/vnode/CMakeLists.txt b/src/vnode/CMakeLists.txt index c953883361ac80d6a962b3b35045792eae48dfae..3fefbea05ba763dfa856dd52c195d36ce70ccd91 100644 --- a/src/vnode/CMakeLists.txt +++ b/src/vnode/CMakeLists.txt @@ -1,6 +1,7 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc) @@ -9,7 +10,5 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) -IF (TD_LINUX) - ADD_LIBRARY(vnode ${SRC}) - TARGET_LINK_LIBRARIES(vnode tsdb tcq) -ENDIF () +ADD_LIBRARY(vnode ${SRC}) +TARGET_LINK_LIBRARIES(vnode tsdb tcq common) diff --git a/src/vnode/inc/vnodeBackup.h b/src/vnode/inc/vnodeBackup.h new file mode 100644 index 0000000000000000000000000000000000000000..0a6b26546c809fd27701bf8194243a2a79561fcb --- /dev/null +++ b/src/vnode/inc/vnodeBackup.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_BACKUP_H +#define TDENGINE_VNODE_BACKUP_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeInitBackup(); +void vnodeCleanupBackup(); +int32_t vnodeBackup(int32_t vgId); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 3ec77bbc122341935a804448eca61258c634cf58..4aa07196a79513c476072b58d62787bd7a5a966d 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -40,6 +40,7 @@ typedef struct { int32_t queuedWMsg; int32_t queuedRMsg; int32_t flowctrlLevel; + int64_t sequence; // for topic int8_t status; int8_t role; int8_t accessState; @@ -47,7 +48,7 @@ typedef struct { int8_t isCommiting; int8_t dbReplica; int8_t dropped; - int8_t reserved; + int8_t dbType; uint64_t version; // current version uint64_t cversion; // version while commit start uint64_t fversion; // version on saved data file diff --git a/src/vnode/inc/vnodeMain.h b/src/vnode/inc/vnodeMain.h index e1ddcdc36aa1fbf434b138f8d6fef5966e1fbc3e..91a5d632cd64d7979b77dcf86472ddf3be2aa1b4 100644 --- a/src/vnode/inc/vnodeMain.h +++ b/src/vnode/inc/vnodeMain.h @@ -25,9 +25,8 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId); int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); +int32_t vnodeSync(int32_t vgId); int32_t vnodeClose(int32_t vgId); - -int32_t vnodeReset(SVnodeObj *pVnode); void vnodeCleanUp(SVnodeObj *pVnode); void vnodeDestroy(SVnodeObj *pVnode); @@ -35,4 +34,4 @@ void vnodeDestroy(SVnodeObj *pVnode); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/vnode/inc/vnodeSync.h b/src/vnode/inc/vnodeSync.h index ae02ca17cb50ed35480916d624004e33c74cc611..28fb63dd6a2db971b430b526b4304a37c3ece9a1 100644 --- a/src/vnode/inc/vnodeSync.h +++ b/src/vnode/inc/vnodeSync.h @@ -25,12 +25,13 @@ uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t ei int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId); void vnodeNotifyRole(int32_t vgId, int8_t role); void vnodeCtrlFlow(int32_t vgId, int32_t level); -int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); +void vnodeStartSyncFile(int32_t vgId); +void vnodeStopSyncFile(int32_t vgId, uint64_t fversion); void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver); -void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code); +void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force); #ifdef __cplusplus } diff --git a/src/vnode/src/vnodeBackup.c b/src/vnode/src/vnodeBackup.c new file mode 100644 index 0000000000000000000000000000000000000000..a0a975be2bcfbb2c945a72adecbf47f9cf404b40 --- /dev/null +++ b/src/vnode/src/vnodeBackup.c @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tutil.h" +#include "tqueue.h" +#include "tglobal.h" +#include "tfs.h" +#include "vnodeBackup.h" +#include "vnodeMain.h" + +typedef struct { + int32_t vgId; +} SVBackupMsg; + +typedef struct { + pthread_t thread; + int32_t workerId; +} SVBackupWorker; + +typedef struct { + int32_t num; + SVBackupWorker *worker; +} SVBackupWorkerPool; + +static SVBackupWorkerPool tsVBackupPool; +static taos_qset tsVBackupQset; +static taos_queue tsVBackupQueue; + +static void vnodeProcessBackupMsg(SVBackupMsg *pMsg) { + int32_t vgId = pMsg->vgId; + char newDir[TSDB_FILENAME_LEN] = {0}; + char stagingDir[TSDB_FILENAME_LEN] = {0}; + + sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); + sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); + + if (tsEnableVnodeBak) { + tfsRmdir(newDir); + tfsRename(stagingDir, newDir); + } else { + vInfo("vgId:%d, vnode backup not enabled", vgId); + + tfsRmdir(stagingDir); + } +} + +static void *vnodeBackupFunc(void *param) { + while (1) { + SVBackupMsg *pMsg = NULL; + if (taosReadQitemFromQset(tsVBackupQset, NULL, (void **)&pMsg, NULL) == 0) { + vDebug("qset:%p, vbackup got no message from qset, exiting", tsVBackupQset); + break; + } + + vTrace("vgId:%d, will be processed in vbackup queue", pMsg->vgId); + vnodeProcessBackupMsg(pMsg); + + vTrace("vgId:%d, disposed in vbackup worker", pMsg->vgId); + taosFreeQitem(pMsg); + } + + return NULL; +} + +static int32_t vnodeStartBackup() { + tsVBackupQueue = taosOpenQueue(); + if (tsVBackupQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; + + taosAddIntoQset(tsVBackupQset, tsVBackupQueue, NULL); + + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + pWorker->workerId = i; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, vnodeBackupFunc, pWorker) != 0) { + vError("failed to create thread to process vbackup queue, reason:%s", strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + + vDebug("vbackup:%d is launched, total:%d", pWorker->workerId, tsVBackupPool.num); + } + + vDebug("vbackup queue:%p is allocated", tsVBackupQueue); + + return TSDB_CODE_SUCCESS; +} + +static int32_t vnodeWriteIntoBackupWorker(int32_t vgId) { + SVBackupMsg *pMsg = taosAllocateQitem(sizeof(SVBackupMsg)); + if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY; + + pMsg->vgId = vgId; + + int32_t code = taosWriteQitem(tsVBackupQueue, TAOS_QTYPE_RPC, pMsg); + if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS; + + return code; +} + +int32_t vnodeBackup(int32_t vgId) { + vTrace("vgId:%d, will backup", vgId); + return vnodeWriteIntoBackupWorker(vgId); +} + +int32_t vnodeInitBackup() { + tsVBackupQset = taosOpenQset(); + + tsVBackupPool.num = 1; + tsVBackupPool.worker = calloc(sizeof(SVBackupWorker), tsVBackupPool.num); + + if (tsVBackupPool.worker == NULL) return -1; + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + pWorker->workerId = i; + vDebug("vbackup:%d is created", i); + } + + vDebug("vbackup is initialized, num:%d qset:%p", tsVBackupPool.num, tsVBackupQset); + + return vnodeStartBackup(); +} + +void vnodeCleanupBackup() { + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + if (taosCheckPthreadValid(pWorker->thread)) { + taosQsetThreadResume(tsVBackupQset); + } + vDebug("vbackup:%d is closed", i); + } + + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + vDebug("vbackup:%d start to join", i); + if (taosCheckPthreadValid(pWorker->thread)) { + pthread_join(pWorker->thread, NULL); + } + vDebug("vbackup:%d join success", i); + } + + vDebug("vbackup is closed, qset:%p", tsVBackupQset); + + taosCloseQset(tsVBackupQset); + tsVBackupQset = NULL; + + tfree(tsVBackupPool.worker); + + vDebug("vbackup queue:%p is freed", tsVBackupQueue); + taosCloseQueue(tsVBackupQueue); + tsVBackupQueue = NULL; +} diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index 0b32f97939a04b695c641a076e863fe37d85a437..c9cd366c6406cb3212fc271f924e1c7b4c42995d 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -34,6 +34,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { pVnode->tsdbCfg.maxRowsPerFileBlock = vnodeMsg->cfg.maxRowsPerFileBlock; pVnode->tsdbCfg.precision = vnodeMsg->cfg.precision; pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; + pVnode->tsdbCfg.update = vnodeMsg->cfg.update; pVnode->tsdbCfg.cacheLastRow = vnodeMsg->cfg.cacheLastRow; pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; @@ -41,6 +42,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { pVnode->syncCfg.replica = vnodeMsg->cfg.vgReplica; pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum; pVnode->dbReplica = vnodeMsg->cfg.dbReplica; + pVnode->dbType = vnodeMsg->cfg.dbType; for (int i = 0; i < pVnode->syncCfg.replica; ++i) { SVnodeDesc *node = &vnodeMsg->nodes[i]; @@ -78,7 +80,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { goto PARSE_VCFG_ERROR; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); goto PARSE_VCFG_ERROR; @@ -103,14 +105,14 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, cfgVersion not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.dbCfgVersion = dbCfgVersion->valueint; + vnodeMsg.cfg.dbCfgVersion = (int32_t)dbCfgVersion->valueint; cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion"); if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) { vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file); vnodeMsg.cfg.vgCfgVersion = 0; } else { - vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint; + vnodeMsg.cfg.vgCfgVersion = (int32_t)vgCfgVersion->valueint; } cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize"); @@ -118,56 +120,56 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, cacheBlockSize not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.cacheBlockSize = cacheBlockSize->valueint; + vnodeMsg.cfg.cacheBlockSize = (int32_t)cacheBlockSize->valueint; cJSON *totalBlocks = cJSON_GetObjectItem(root, "totalBlocks"); if (!totalBlocks || totalBlocks->type != cJSON_Number) { vError("vgId:%d, failed to read %s, totalBlocks not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.totalBlocks = totalBlocks->valueint; + vnodeMsg.cfg.totalBlocks = (int32_t)totalBlocks->valueint; cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysPerFile not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysPerFile = daysPerFile->valueint; + vnodeMsg.cfg.daysPerFile = (int32_t)daysPerFile->valueint; cJSON *daysToKeep = cJSON_GetObjectItem(root, "daysToKeep"); if (!daysToKeep || daysToKeep->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep = daysToKeep->valueint; + vnodeMsg.cfg.daysToKeep = (int32_t)daysToKeep->valueint; cJSON *daysToKeep1 = cJSON_GetObjectItem(root, "daysToKeep1"); if (!daysToKeep1 || daysToKeep1->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep1 not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep1 = daysToKeep1->valueint; + vnodeMsg.cfg.daysToKeep1 = (int32_t)daysToKeep1->valueint; cJSON *daysToKeep2 = cJSON_GetObjectItem(root, "daysToKeep2"); if (!daysToKeep2 || daysToKeep2->type != cJSON_Number) { vError("vgId:%d, failed to read %s, daysToKeep2 not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.daysToKeep2 = daysToKeep2->valueint; + vnodeMsg.cfg.daysToKeep2 = (int32_t)daysToKeep2->valueint; cJSON *minRowsPerFileBlock = cJSON_GetObjectItem(root, "minRowsPerFileBlock"); if (!minRowsPerFileBlock || minRowsPerFileBlock->type != cJSON_Number) { vError("vgId:%d, failed to read %s, minRowsPerFileBlock not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.minRowsPerFileBlock = minRowsPerFileBlock->valueint; + vnodeMsg.cfg.minRowsPerFileBlock = (int32_t)minRowsPerFileBlock->valueint; cJSON *maxRowsPerFileBlock = cJSON_GetObjectItem(root, "maxRowsPerFileBlock"); if (!maxRowsPerFileBlock || maxRowsPerFileBlock->type != cJSON_Number) { vError("vgId:%d, failed to read %s, maxRowsPerFileBlock not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; + vnodeMsg.cfg.maxRowsPerFileBlock = (int32_t)maxRowsPerFileBlock->valueint; cJSON *precision = cJSON_GetObjectItem(root, "precision"); if (!precision || precision->type != cJSON_Number) { @@ -195,7 +197,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, fsyncPeriod not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - vnodeMsg.cfg.fsyncPeriod = fsyncPeriod->valueint; + vnodeMsg.cfg.fsyncPeriod = (int32_t)fsyncPeriod->valueint; cJSON *wals = cJSON_GetObjectItem(root, "wals"); if (!wals || wals->type != cJSON_Number) { @@ -213,7 +215,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { cJSON *dbReplica = cJSON_GetObjectItem(root, "dbReplica"); if (!dbReplica || dbReplica->type != cJSON_Number) { - vError("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file); + vWarn("vgId:%d, failed to read %s, dbReplica not found", pVnode->vgId, file); vnodeMsg.cfg.dbReplica = vnodeMsg.cfg.vgReplica; vnodeMsg.cfg.vgCfgVersion = 0; } else { @@ -227,15 +229,32 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } vnodeMsg.cfg.quorum = (int8_t)quorum->valueint; + cJSON *update = cJSON_GetObjectItem(root, "update"); + if (!update || update->type != cJSON_Number) { + vWarn("vgId: %d, failed to read %s, update not found", pVnode->vgId, file); + vnodeMsg.cfg.update = 0; + vnodeMsg.cfg.vgCfgVersion = 0; + } else { + vnodeMsg.cfg.update = (int8_t)update->valueint; + } + cJSON *cacheLastRow = cJSON_GetObjectItem(root, "cacheLastRow"); if (!cacheLastRow || cacheLastRow->type != cJSON_Number) { - vError("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file); + vWarn("vgId: %d, failed to read %s, cacheLastRow not found", pVnode->vgId, file); vnodeMsg.cfg.cacheLastRow = 0; vnodeMsg.cfg.vgCfgVersion = 0; } else { vnodeMsg.cfg.cacheLastRow = (int8_t)cacheLastRow->valueint; } + cJSON *dbType = cJSON_GetObjectItem(root, "dbType"); + if (!dbType || dbType->type != cJSON_Number) { + vWarn("vgId: %d, failed to read %s, dbType not found", pVnode->vgId, file); + vnodeMsg.cfg.dbType = 0; + } else { + vnodeMsg.cfg.dbType = (int8_t)dbType->valueint; + } + cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { vError("vgId:%d, failed to read %s, nodeInfos not found", pVnode->vgId, file); @@ -258,7 +277,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, nodeId not found", pVnode->vgId, file); goto PARSE_VCFG_ERROR; } - node->nodeId = nodeId->valueint; + node->nodeId = (int32_t)nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { @@ -325,7 +344,9 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, " \"dbReplica\": %d,\n", pMsg->cfg.dbReplica); len += snprintf(content + len, maxLen - len, " \"wals\": %d,\n", pMsg->cfg.wals); len += snprintf(content + len, maxLen - len, " \"quorum\": %d,\n", pMsg->cfg.quorum); + len += snprintf(content + len, maxLen - len, " \"update\": %d,\n", pMsg->cfg.update); len += snprintf(content + len, maxLen - len, " \"cacheLastRow\": %d,\n", pMsg->cfg.cacheLastRow); + len += snprintf(content + len, maxLen - len, " \"dbType\": %d,\n", pMsg->cfg.dbType); len += snprintf(content + len, maxLen - len, " \"nodeInfos\": [{\n"); for (int32_t i = 0; i < pMsg->cfg.vgReplica; i++) { SVnodeDesc *node = &pMsg->nodes[i]; @@ -341,7 +362,7 @@ int32_t vnodeWriteCfg(SCreateVnodeMsg *pMsg) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 5f6f3fe105a9f0e4933db830dc42a6b597ea2ee1..69d94aff87388a96664b40b7db1d1c1e906ca8b2 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -18,7 +18,7 @@ #include "taoserror.h" #include "taosmsg.h" #include "tglobal.h" -// #include "tfs.h" +#include "tfs.h" #include "query.h" #include "dnode.h" #include "vnodeCfg.h" @@ -27,6 +27,7 @@ #include "vnodeVersion.h" #include "vnodeMgmt.h" #include "vnodeWorker.h" +#include "vnodeBackup.h" #include "vnodeMain.h" static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); @@ -41,32 +42,19 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { return TSDB_CODE_SUCCESS; } - if (mkdir(tsVnodeDir, 0755) != 0 && errno != EEXIST) { - vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), tsVnodeDir); - if (errno == EACCES) { - return TSDB_CODE_VND_NO_DISK_PERMISSIONS; - } else if (errno == ENOSPC) { - return TSDB_CODE_VND_NO_DISKSPACE; - } else if (errno == ENOENT) { - return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR; - } else { - return TSDB_CODE_VND_INIT_FAILED; - } + if (tfsMkdir("vnode") < 0) { + vError("vgId:%d, failed to create vnode dir, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); + return terrno; } char rootDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId); - if (mkdir(rootDir, 0755) != 0 && errno != EEXIST) { - vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), rootDir); - if (errno == EACCES) { - return TSDB_CODE_VND_NO_DISK_PERMISSIONS; - } else if (errno == ENOSPC) { - return TSDB_CODE_VND_NO_DISKSPACE; - } else if (errno == ENOENT) { - return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR; - } else { - return TSDB_CODE_VND_INIT_FAILED; - } + + char vnodeDir[TSDB_FILENAME_LEN] = "\0"; + snprintf(vnodeDir, TSDB_FILENAME_LEN, "/vnode/vnode%d", pVnodeCfg->cfg.vgId); + if (tfsMkdir(vnodeDir) < 0) { + vError("vgId:%d, failed to create vnode dir %s, reason:%s", pVnodeCfg->cfg.vgId, vnodeDir, strerror(errno)); + return terrno; } code = vnodeWriteCfg(pVnodeCfg); @@ -75,22 +63,24 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { return code; } - STsdbCfg tsdbCfg = {0}; - tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; - tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; - tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; - tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; - tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; - tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; - tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; - tsdbCfg.precision = pVnodeCfg->cfg.precision; - tsdbCfg.compression = pVnodeCfg->cfg.compression; - tsdbCfg.update = pVnodeCfg->cfg.update; - tsdbCfg.cacheLastRow = pVnodeCfg->cfg.cacheLastRow; - - char tsdbDir[TSDB_FILENAME_LEN] = {0}; - sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); - if (tsdbCreateRepo(tsdbDir, &tsdbCfg) < 0) { + // STsdbCfg tsdbCfg = {0}; + // tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; + // tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; + // tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; + // tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; + // tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; + // tsdbCfg.keep1 = pVnodeCfg->cfg.daysToKeep1; + // tsdbCfg.keep2 = pVnodeCfg->cfg.daysToKeep2; + // tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; + // tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; + // tsdbCfg.precision = pVnodeCfg->cfg.precision; + // tsdbCfg.compression = pVnodeCfg->cfg.compression; + // tsdbCfg.update = pVnodeCfg->cfg.update; + // tsdbCfg.cacheLastRow = pVnodeCfg->cfg.cacheLastRow; + + // char tsdbDir[TSDB_FILENAME_LEN] = {0}; + // sprintf(tsdbDir, "vnode/vnode%d/tsdb", pVnodeCfg->cfg.vgId); + if (tsdbCreateRepo(pVnodeCfg->cfg.vgId) < 0) { vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); return TSDB_CODE_VND_INIT_FAILED; } @@ -102,6 +92,23 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { return code; } +int32_t vnodeSync(int32_t vgId) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vDebug("vgId:%d, failed to sync, vnode not find", vgId); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } + + if (pVnode->role != TAOS_SYNC_ROLE_MASTER) { + vInfo("vgId:%d, vnode will sync, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); + syncRecover(pVnode->sync); + } + + vnodeRelease(pVnode); + + return TSDB_CODE_SUCCESS; +} + int32_t vnodeDrop(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { @@ -121,31 +128,78 @@ int32_t vnodeDrop(int32_t vgId) { } static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) { + STsdbCfg tsdbCfg = pVnode->tsdbCfg; + SSyncCfg syncCfg = pVnode->syncCfg; + int32_t dbCfgVersion = pVnode->dbCfgVersion; + int32_t vgCfgVersion = pVnode->vgCfgVersion; + int32_t code = vnodeWriteCfg(pVnodeCfg); if (code != TSDB_CODE_SUCCESS) { - return code; + pVnode->dbCfgVersion = dbCfgVersion; + pVnode->vgCfgVersion = vgCfgVersion; + pVnode->syncCfg = syncCfg; + pVnode->tsdbCfg = tsdbCfg; + return code; } code = vnodeReadCfg(pVnode); if (code != TSDB_CODE_SUCCESS) { - return code; + pVnode->dbCfgVersion = dbCfgVersion; + pVnode->vgCfgVersion = vgCfgVersion; + pVnode->syncCfg = syncCfg; + pVnode->tsdbCfg = tsdbCfg; + return code; } code = walAlter(pVnode->wal, &pVnode->walCfg); if (code != TSDB_CODE_SUCCESS) { + pVnode->dbCfgVersion = dbCfgVersion; + pVnode->vgCfgVersion = vgCfgVersion; + pVnode->syncCfg = syncCfg; + pVnode->tsdbCfg = tsdbCfg; return code; } - code = syncReconfig(pVnode->sync, &pVnode->syncCfg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + bool tsdbCfgChanged = (memcmp(&tsdbCfg, &pVnode->tsdbCfg, sizeof(STsdbCfg)) != 0); + bool syncCfgChanged = (memcmp(&syncCfg, &pVnode->syncCfg, sizeof(SSyncCfg)) != 0); - if (pVnode->tsdb) { - code = tsdbConfigRepo(pVnode->tsdb, &pVnode->tsdbCfg); + vDebug("vgId:%d, tsdbchanged:%d syncchanged:%d while alter vnode", pVnode->vgId, tsdbCfgChanged, syncCfgChanged); + + if (/*tsdbCfgChanged || */syncCfgChanged) { + // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS + // dbCfgVersion can be corrected by status msg + if (!vnodeSetUpdatingStatus(pVnode)) { + vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId); + pVnode->dbCfgVersion = dbCfgVersion; + pVnode->vgCfgVersion = vgCfgVersion; + pVnode->syncCfg = syncCfg; + pVnode->tsdbCfg = tsdbCfg; + return TSDB_CODE_SUCCESS; + } + + code = syncReconfig(pVnode->sync, &pVnode->syncCfg); if (code != TSDB_CODE_SUCCESS) { + pVnode->dbCfgVersion = dbCfgVersion; + pVnode->vgCfgVersion = vgCfgVersion; + pVnode->syncCfg = syncCfg; + pVnode->tsdbCfg = tsdbCfg; + vnodeSetReadyStatus(pVnode); return code; } + + if (pVnode->tsdb) { + code = tsdbConfigRepo(pVnode->tsdb, &pVnode->tsdbCfg); + if (code != TSDB_CODE_SUCCESS) { + pVnode->dbCfgVersion = dbCfgVersion; + pVnode->vgCfgVersion = vgCfgVersion; + pVnode->syncCfg = syncCfg; + pVnode->tsdbCfg = tsdbCfg; + vnodeSetReadyStatus(pVnode); + return code; + } + } + + vnodeSetReadyStatus(pVnode); } return 0; @@ -153,21 +207,16 @@ static int32_t vnodeAlterImp(SVnodeObj *pVnode, SCreateVnodeMsg *pVnodeCfg) { int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) { SVnodeObj *pVnode = vparam; - if (pVnode->dbCfgVersion == pVnodeCfg->cfg.dbCfgVersion && pVnode->vgCfgVersion == pVnodeCfg->cfg.vgCfgVersion) { - vDebug("vgId:%d, dbCfgVersion:%d and vgCfgVersion:%d not change", pVnode->vgId, pVnode->dbCfgVersion, - pVnode->vgCfgVersion); - return TSDB_CODE_SUCCESS; - } - // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS - // dbCfgVersion can be corrected by status msg - if (!vnodeSetUpdatingStatus(pVnode)) { - vDebug("vgId:%d, vnode is not ready, do alter operation later", pVnode->vgId); + vDebug("vgId:%d, current dbCfgVersion:%d vgCfgVersion:%d, input dbCfgVersion:%d vgCfgVersion:%d", pVnode->vgId, + pVnode->dbCfgVersion, pVnode->vgCfgVersion, pVnodeCfg->cfg.dbCfgVersion, pVnodeCfg->cfg.vgCfgVersion); + + if (pVnode->dbCfgVersion == pVnodeCfg->cfg.dbCfgVersion && pVnode->vgCfgVersion == pVnodeCfg->cfg.vgCfgVersion) { + vDebug("vgId:%d, cfg not change", pVnode->vgId); return TSDB_CODE_SUCCESS; } int32_t code = vnodeAlterImp(pVnode, pVnodeCfg); - vnodeSetReadyStatus(pVnode); if (code != 0) { vError("vgId:%d, failed to alter vnode, code:0x%x", pVnode->vgId, code); @@ -201,14 +250,18 @@ int32_t vnodeOpen(int32_t vgId) { pthread_mutex_init(&pVnode->statusMutex, NULL); vnodeSetInitStatus(pVnode); + tsdbIncCommitRef(pVnode->vgId); + int32_t code = vnodeReadCfg(pVnode); if (code != TSDB_CODE_SUCCESS) { + vError("vgId:%d, failed to read config file, set cfgVersion to 0", pVnode->vgId); vnodeCleanUp(pVnode); - return code; + return 0; } code = vnodeReadVersion(pVnode); if (code != TSDB_CODE_SUCCESS) { + pVnode->version = 0; vError("vgId:%d, failed to read file version, generate it from data file", pVnode->vgId); // Allow vnode start even when read file version fails, set file version as wal version or zero // vnodeCleanUp(pVnode); @@ -245,14 +298,13 @@ int32_t vnodeOpen(int32_t vgId) { appH.cqH = pVnode->cq; appH.cqCreateFunc = cqCreate; appH.cqDropFunc = cqDrop; - sprintf(temp, "%s/tsdb", rootDir); terrno = 0; - pVnode->tsdb = tsdbOpenRepo(temp, &appH); + pVnode->tsdb = tsdbOpenRepo(&(pVnode->tsdbCfg), &appH); if (pVnode->tsdb == NULL) { vnodeCleanUp(pVnode); return terrno; - } else if (terrno != TSDB_CODE_SUCCESS) { + } else if (tsdbGetState(pVnode->tsdb) != TSDB_STATE_OK) { vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); if (pVnode->syncCfg.replica <= 1) { @@ -297,23 +349,25 @@ int32_t vnodeOpen(int32_t vgId) { pVnode->events = NULL; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); - tsdbIncCommitRef(pVnode->vgId); vnodeAddIntoHash(pVnode); - + SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; syncInfo.version = pVnode->version; syncInfo.syncCfg = pVnode->syncCfg; tstrncpy(syncInfo.path, rootDir, TSDB_FILENAME_LEN); - syncInfo.getWalInfo = vnodeGetWalInfo; - syncInfo.getFileInfo = vnodeGetFileInfo; - syncInfo.writeToCache = vnodeWriteToCache; + syncInfo.getWalInfoFp = vnodeGetWalInfo; + syncInfo.writeToCacheFp = vnodeWriteToCache; syncInfo.confirmForward = vnodeConfirmForard; - syncInfo.notifyRole = vnodeNotifyRole; - syncInfo.notifyFlowCtrl = vnodeCtrlFlow; - syncInfo.notifyFileSynced = vnodeNotifyFileSynced; - syncInfo.getVersion = vnodeGetVersion; + syncInfo.notifyRoleFp = vnodeNotifyRole; + syncInfo.notifyFlowCtrlFp = vnodeCtrlFlow; + syncInfo.startSyncFileFp = vnodeStartSyncFile; + syncInfo.stopSyncFileFp = vnodeStopSyncFile; + syncInfo.getVersionFp = vnodeGetVersion; + syncInfo.sendFileFp = tsdbSyncSend; + syncInfo.recvFileFp = tsdbSyncRecv; + syncInfo.pTsdb = pVnode->tsdb; pVnode->sync = syncStart(&syncInfo); if (pVnode->sync <= 0) { @@ -343,7 +397,7 @@ int32_t vnodeClose(int32_t vgId) { void vnodeDestroy(SVnodeObj *pVnode) { int32_t code = 0; int32_t vgId = pVnode->vgId; - + if (pVnode->qMgmt) { qCleanupQueryMgmt(pVnode->qMgmt); pVnode->qMgmt = NULL; @@ -394,18 +448,14 @@ void vnodeDestroy(SVnodeObj *pVnode) { if (pVnode->dropped) { char rootDir[TSDB_FILENAME_LEN] = {0}; - char newDir[TSDB_FILENAME_LEN] = {0}; - sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId); - sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId); + char stagingDir[TSDB_FILENAME_LEN] = {0}; + sprintf(rootDir, "%s/vnode%d", "vnode", vgId); + sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); - if (0 == tsEnableVnodeBak) { - vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId); - } else { - taosRemoveDir(newDir); - taosRename(rootDir, newDir); - } + tfsRename(rootDir, stagingDir); + + vnodeBackup(vgId); - taosRemoveDir(rootDir); dnodeSendStatusMsgToMnode(); } @@ -445,7 +495,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { if (status == TSDB_STATUS_COMMIT_START) { pVnode->isCommiting = 1; pVnode->cversion = pVnode->version; - vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); + vInfo("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); if (!vnodeInInitStatus(pVnode)) { return walRenew(pVnode->wal); } @@ -456,7 +506,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { pVnode->isCommiting = 0; pVnode->isFull = 0; pVnode->fversion = pVnode->cversion; - vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); + vInfo("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); if (!vnodeInInitStatus(pVnode)) { walRemoveOneOldFile(pVnode->wal); } @@ -465,37 +515,3 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { return 0; } - -int32_t vnodeReset(SVnodeObj *pVnode) { - char rootDir[128] = "\0"; - sprintf(rootDir, "%s/tsdb", pVnode->rootDir); - - if (!vnodeSetResetStatus(pVnode)) { - return -1; - } - - void *tsdb = pVnode->tsdb; - pVnode->tsdb = NULL; - - // acquire vnode - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - - if (refCount > 3) { - tsem_wait(&pVnode->sem); - } - - // close tsdb, then open tsdb - tsdbCloseRepo(tsdb, 0); - STsdbAppH appH = {0}; - appH.appH = (void *)pVnode; - appH.notifyStatus = vnodeProcessTsdbStatus; - appH.cqH = pVnode->cq; - appH.cqCreateFunc = cqCreate; - appH.cqDropFunc = cqDrop; - pVnode->tsdb = tsdbOpenRepo(rootDir, &appH); - - vnodeSetReadyStatus(pVnode); - vnodeRelease(pVnode); - - return 0; -} diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 8469ab12c1ef5833e73058c167afb2777ac76ba9..32f95321383981924c5b6496bd4302edca19da5e 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -17,6 +17,7 @@ #include "os.h" #include "dnode.h" #include "vnodeStatus.h" +#include "vnodeBackup.h" #include "vnodeWorker.h" #include "vnodeRead.h" #include "vnodeWrite.h" @@ -29,6 +30,7 @@ static void vnodeCleanupHash(void); static void vnodeIncRef(void *ptNode); static SStep tsVnodeSteps[] = { + {"vnode-backup", vnodeInitBackup, vnodeCleanupBackup}, {"vnode-worker", vnodeInitMWorker, vnodeCleanupMWorker}, {"vnode-write", vnodeInitWrite, vnodeCleanupWrite}, {"vnode-read", vnodeInitRead, vnodeCleanupRead}, @@ -128,7 +130,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { int64_t compStorage = 0; int64_t pointsWritten = 0; - if (!vnodeInReadyStatus(pVnode)) return; + if (vnodeInClosingStatus(pVnode)) return; if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; if (pVnode->tsdb) { @@ -194,4 +196,4 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { vnodeRelease(pVnode); } } -} \ No newline at end of file +} diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index acb1fc78f6554dc7cd4b984107a7bc1b95315b67..0836ade77ff6617ee032e7266c99ff73088f221b 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -25,7 +25,7 @@ static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SV static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead); static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead); -static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId); +static int32_t vnodeNotifyCurrentQhandle(void* handle, uint64_t qId, void* qhandle, int32_t vgId); int32_t vnodeInitRead(void) { vnodeProcessReadMsgFp[TSDB_MSG_TYPE_QUERY] = vnodeProcessQueryMsg; @@ -167,7 +167,7 @@ static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void * @param ahandle sqlObj address at client side * @return */ -static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, bool *freeHandle, void *ahandle) { +static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, uint64_t qId, void **handle, bool *freeHandle, void *ahandle) { bool continueExec = false; int32_t code = TSDB_CODE_SUCCESS; @@ -183,7 +183,7 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, } } else { *freeHandle = true; - vTrace("QInfo:%p exec completed, free handle:%d", *handle, *freeHandle); + vTrace("QInfo:%"PRIu64"-%p exec completed, free handle:%d", qId, *handle, *freeHandle); } } else { SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -220,34 +220,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { vError("error rpc msg in query, %s", tstrerror(pRead->code)); } -// assert(pRead->code != TSDB_CODE_RPC_NETWORK_UNAVAIL); -// if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { -// SCancelQueryMsg *pMsg = (SCancelQueryMsg *)pRead->pCont; -//// pMsg->free = htons(killQueryMsg->free); -// pMsg->qhandle = htobe64(pMsg->qhandle); -// -// vWarn("QInfo:%p connection %p broken, kill query", (void *)pMsg->qhandle, pRead->rpcHandle); -//// assert(pRead->contLen > 0 && pMsg->free == 1); -// -// void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)pMsg->qhandle); -// if (qhandle == NULL || *qhandle == NULL) { -// vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)pMsg->qhandle, pRead->rpcHandle); -// } else { -// assert(*qhandle == (void *)pMsg->qhandle); -// -// qKillQuery(*qhandle); -// qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, true); -// } -// -// return TSDB_CODE_TSC_QUERY_CANCELLED; -// } int32_t code = TSDB_CODE_SUCCESS; void ** handle = NULL; if (contLen != 0) { qinfo_t pQInfo = NULL; - code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo); + uint64_t qId = 0; + code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, &qId); SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp)); pRsp->code = code; @@ -259,22 +239,22 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // current connect is broken if (code == TSDB_CODE_SUCCESS) { - handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t)pQInfo); + handle = qRegisterQInfo(pVnode->qMgmt, qId, (uint64_t)pQInfo); if (handle == NULL) { // failed to register qhandle pRsp->code = terrno; terrno = 0; - vError("vgId:%d, QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo, + vError("vgId:%d, QInfo:%"PRIu64 "-%p register qhandle failed, return to app, code:%s", pVnode->vgId, qId, (void *)pQInfo, tstrerror(pRsp->code)); qDestroyQueryInfo(pQInfo); // destroy it directly return pRsp->code; } else { assert(*handle == pQInfo); - pRsp->qhandle = htobe64((uint64_t)pQInfo); + pRsp->qhandle = htobe64(qId); } if (handle != NULL && - vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, + vnodeNotifyCurrentQhandle(pRead->rpcHandle, qId, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%"PRIu64 "-%p, query discarded since link is broken, %p", pVnode->vgId, qId, *handle, pRead->rpcHandle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -285,7 +265,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } if (handle != NULL) { - vTrace("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); + vTrace("vgId:%d, QInfo:%"PRIu64 "-%p, dnode query msg disposed, create qhandle and returns to app", vgId, qId, *handle); code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle); if (code != TSDB_CODE_SUCCESS) { pRsp->code = code; @@ -296,16 +276,17 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } else { assert(pCont != NULL); void **qhandle = (void **)pRead->qhandle; + uint64_t qId = 0; vTrace("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); // In the retrieve blocking model, only 50% CPU will be used in query processing if (tsRetrieveBlockingModel) { - qTableQuery(*qhandle); // do execute query + qTableQuery(*qhandle, &qId); // do execute query qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); } else { bool freehandle = false; - bool buildRes = qTableQuery(*qhandle); // do execute query + bool buildRes = qTableQuery(*qhandle, &qId); // do execute query // build query rsp, the retrieve request has reached here already if (buildRes) { @@ -317,7 +298,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { pRead->rpcHandle); // set the real rsp error code - pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle); + pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qId, qhandle, &freehandle, pRead->rpcHandle); // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; @@ -347,32 +328,32 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { SRetrieveTableMsg *pRetrieve = pCont; pRetrieve->free = htons(pRetrieve->free); - pRetrieve->qhandle = htobe64(pRetrieve->qhandle); + pRetrieve->qId = htobe64(pRetrieve->qId); - vTrace("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, + vTrace("vgId:%d, qId:%" PRIu64 ", retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, pRetrieve->qId, pRetrieve->free, pRead->rpcHandle); memset(pRet, 0, sizeof(SRspRet)); terrno = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS; - void ** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle); + void ** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qId); if (handle == NULL) { code = terrno; terrno = TSDB_CODE_SUCCESS; - } else if ((*handle) != (void *)pRetrieve->qhandle) { + } else if (!checkQIdEqual(*handle, pRetrieve->qId)) { code = TSDB_CODE_QRY_INVALID_QHANDLE; } if (code != TSDB_CODE_SUCCESS) { - vError("vgId:%d, invalid handle in retrieving result, code:%s, QInfo:%p", pVnode->vgId, tstrerror(code), (void *)pRetrieve->qhandle); + vError("vgId:%d, invalid qId in retrieving result, code:%s, QInfo:%" PRIu64, pVnode->vgId, tstrerror(code), pRetrieve->qId); vnodeBuildNoResultQueryRsp(pRet); return code; } // kill current query and free corresponding resources. if (pRetrieve->free == 1) { - vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); + vWarn("vgId:%d, QInfo:%"PRIu64 "-%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, pRetrieve->qId, *handle); qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -382,8 +363,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } // register the qhandle to connect to quit query immediate if connection is broken - if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pRead->rpcHandle); + if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, pRetrieve->qId, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%"PRIu64 "-%p, retrieve discarded since link is broken, conn:%p", pVnode->vgId, pRetrieve->qhandle, *handle, pRead->rpcHandle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -412,7 +393,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { } // ahandle is the sqlObj pointer - code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle); + code = vnodeDumpQueryResult(pRet, pVnode, pRetrieve->qId, handle, &freeHandle, pRead->rpcHandle); } // If qhandle is not added into vread queue, the query should be completed already or paused with error. @@ -426,13 +407,13 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // notify connection(handle) that current qhandle is created, if current connection from // client is broken, the query needs to be killed immediately. -int32_t vnodeNotifyCurrentQhandle(void *handle, void *qhandle, int32_t vgId) { +int32_t vnodeNotifyCurrentQhandle(void *handle, uint64_t qId, void *qhandle, int32_t vgId) { SRetrieveTableMsg *pMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); - pMsg->qhandle = htobe64((uint64_t)qhandle); + pMsg->qId = htobe64(qId); pMsg->header.vgId = htonl(vgId); pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); - vTrace("QInfo:%p register qhandle to connect:%p", qhandle, handle); + vTrace("QInfo:%"PRIu64"-%p register qhandle to connect:%p", qId, qhandle, handle); return rpcReportProgress(handle, (char *)pMsg, sizeof(SRetrieveTableMsg)); } @@ -441,4 +422,4 @@ void vnodeWaitReadCompleted(SVnodeObj *pVnode) { vTrace("vgId:%d, queued rmsg num:%d", pVnode->vgId, pVnode->queuedRMsg); taosMsleep(10); } -} \ No newline at end of file +} diff --git a/src/vnode/src/vnodeStatus.c b/src/vnode/src/vnodeStatus.c index 68898435303daa8e0ac60a84cd54f4794515103e..c482d1fd1a44497b2ba3ff8482d4d1f66bb11ff5 100644 --- a/src/vnode/src/vnodeStatus.c +++ b/src/vnode/src/vnodeStatus.c @@ -109,6 +109,8 @@ bool vnodeSetResetStatus(SVnodeObj* pVnode) { taosMsleep(1); } + vInfo("vgId:%d, set to reset status", pVnode->vgId); + // release local resources only after cutting off outside connections qQueryMgmtNotifyClosed(pVnode->qMgmt); vnodeWaitReadCompleted(pVnode); @@ -153,6 +155,18 @@ bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode) { return in; } +bool vnodeInClosingStatus(SVnodeObj* pVnode) { + bool in = false; + pthread_mutex_lock(&pVnode->statusMutex); + + if (pVnode->status == TAOS_VN_STATUS_CLOSING) { + in = true; + } + + pthread_mutex_unlock(&pVnode->statusMutex); + return in; +} + bool vnodeInResetStatus(SVnodeObj* pVnode) { bool in = false; pthread_mutex_lock(&pVnode->statusMutex); diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index c67132c41f2a925c5d3224f8b59f0e94b5e8f1c3..aa4cf0fc15bd9db749888699baf90a97bfbeb39f 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -20,6 +20,7 @@ #include "dnode.h" #include "vnodeVersion.h" #include "vnodeMain.h" +#include "vnodeStatus.h" uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver) { SVnodeObj *pVnode = vnodeAcquire(vgId); @@ -83,22 +84,35 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) { vnodeRelease(pVnode); } -int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) { +void vnodeStartSyncFile(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { - vError("vgId:%d, vnode not found while notify file synced", vgId); - return 0; + vError("vgId:%d, vnode not found while start filesync", vgId); + return; + } + + vInfo("vgId:%d, datafile will be synced", vgId); + vnodeSetResetStatus(pVnode); + + vnodeRelease(pVnode); +} + +void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while stop filesync", vgId); + return; } pVnode->fversion = fversion; pVnode->version = fversion; vnodeSaveVersion(pVnode); + walResetVersion(pVnode->wal, fversion); - vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); - int32_t code = vnodeReset(pVnode); + vInfo("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); + vnodeSetReadyStatus(pVnode); vnodeRelease(pVnode); - return code; } void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) { @@ -134,7 +148,7 @@ int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { int32_t code = 0; if (pVnode->isCommiting) { - vDebug("vgId:%d, vnode is commiting while get version", vgId); + vInfo("vgId:%d, vnode is commiting while get version", vgId); code = -1; } else { *fver = pVnode->fversion; @@ -145,7 +159,7 @@ int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { return code; } -void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code) { +void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code, bool force) { SVnodeObj *pVnode = vparam; - syncConfirmForward(pVnode->sync, version, code); -} + syncConfirmForward(pVnode->sync, version, code, force); +} \ No newline at end of file diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c index fb3b3ebd9e9b9b4e6ac4d3f62de3b7599a13cae9..d22bc17cbeda570f119a5a6e67637382f3a25bf8 100644 --- a/src/vnode/src/vnodeVersion.c +++ b/src/vnode/src/vnodeVersion.c @@ -41,7 +41,7 @@ int32_t vnodeReadVersion(SVnodeObj *pVnode) { goto PARSE_VER_ERROR; } - len = fread(content, 1, maxLen, fp); + len = (int32_t)fread(content, 1, maxLen, fp); if (len <= 0) { vError("vgId:%d, failed to read %s, content is null", pVnode->vgId, file); goto PARSE_VER_ERROR; @@ -90,7 +90,7 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) { len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); - fflush(fp); + fsync(fileno(fp)); fclose(fp); free(content); terrno = 0; diff --git a/src/vnode/src/vnodeWorker.c b/src/vnode/src/vnodeWorker.c index d6053cf18e2569a197c4bdf96ff6394ede716e42..6fb79d10feef74d88e04368c5607d39e7d70f6f2 100644 --- a/src/vnode/src/vnodeWorker.c +++ b/src/vnode/src/vnodeWorker.c @@ -109,7 +109,7 @@ static void vnodeStopMWorker() { void vnodeCleanupMWorker() { for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { SVMWorker *pWorker = tsVMWorkerPool.worker + i; - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { taosQsetThreadResume(tsVMWorkerQset); } vDebug("vmworker:%d is closed", i); @@ -118,7 +118,7 @@ void vnodeCleanupMWorker() { for (int32_t i = 0; i < tsVMWorkerPool.maxNum; ++i) { SVMWorker *pWorker = tsVMWorkerPool.worker + i; vDebug("vmworker:%d start to join", i); - if (pWorker->thread) { + if (taosCheckPthreadValid(pWorker->thread)) { pthread_join(pWorker->thread, NULL); } vDebug("vmworker:%d join success", i); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 801d51b3c80c25bfaeec83320b979640478db295..92e1ba804be5ab30366594933ebcdaff3bd0727b 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "tp.h" #include "taosmsg.h" #include "taoserror.h" #include "tglobal.h" @@ -23,7 +24,7 @@ #include "dnode.h" #include "vnodeStatus.h" -#define MAX_QUEUED_MSG_NUM 10000 +#define MAX_QUEUED_MSG_NUM 100000 extern void * tsDnodeTmr; static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, SRspRet *); @@ -88,12 +89,14 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara // forward to peers, even it is WAL/FWD, it shall be called to update version in sync int32_t syncCode = 0; - syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype); + bool force = (pWrite == NULL ? false : pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT); + syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype, force); if (syncCode < 0) return syncCode; // write into WAL code = walWrite(pVnode->wal, pHead); if (code < 0) { + if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1); vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code); return code; } @@ -102,7 +105,10 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara // write data locally code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet); - if (code < 0) return code; + if (code < 0) { + if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1); + return code; + } return syncCode; } @@ -139,6 +145,10 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR vTrace("vgId:%d, submit msg is processed", pVnode->vgId); + if (pVnode->dbType == TSDB_DB_TYPE_TOPIC && pVnode->role == TAOS_SYNC_ROLE_MASTER) { + tpUpdateTs(pVnode->vgId, &pVnode->sequence, pCont); + } + // save insert result into item SShellSubmitRspMsg *pRsp = NULL; if (pRet) { @@ -174,7 +184,7 @@ static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet SMDDropTableMsg *pTable = pCont; int32_t code = TSDB_CODE_SUCCESS; - vDebug("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableId); + vDebug("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableFname); STableId tableId = {.uid = htobe64(pTable->uid), .tid = htonl(pTable->tid)}; if (tsdbDropTable(pVnode->tsdb, tableId) < 0) code = terrno; @@ -197,13 +207,13 @@ static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet SDropSTableMsg *pTable = pCont; int32_t code = TSDB_CODE_SUCCESS; - vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId); + vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableFname); STableId stableId = {.uid = htobe64(pTable->uid), .tid = -1}; if (tsdbDropTable(pVnode->tsdb, stableId) < 0) code = terrno; - vDebug("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableId, tstrerror(code)); + vDebug("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableFname, tstrerror(code)); return code; } @@ -333,7 +343,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0; if (tsEnableFlowCtrl == 0) { - int32_t ms = pow(2, pVnode->flowctrlLevel + 2); + int32_t ms = (int32_t)pow(2, pVnode->flowctrlLevel + 2); if (ms > 100) ms = 100; vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); taosMsleep(ms); diff --git a/src/wal/CMakeLists.txt b/src/wal/CMakeLists.txt index 681bed54258353b7ceff6df3c84909cbcadaa6b8..a89024dab5060b1f18174f769e0d70c00ad00faf 100644 --- a/src/wal/CMakeLists.txt +++ b/src/wal/CMakeLists.txt @@ -1,11 +1,11 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) +ADD_DEFINITIONS(-DWAL_CHECKSUM_WHOLE) + INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) -IF (TD_LINUX) - ADD_LIBRARY(twal ${SRC}) - TARGET_LINK_LIBRARIES(twal tutil common) - ADD_SUBDIRECTORY(test) -ENDIF () +ADD_LIBRARY(twal ${SRC}) +TARGET_LINK_LIBRARIES(twal tutil common) +ADD_SUBDIRECTORY(test) diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 72ea239817778db036e1eb05fc67fdcd128cbb6b..39ce2657aa2c27c2249d1492f5dbcde2517cc988 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -44,7 +44,7 @@ int32_t walInit() { return code; } - wInfo("wal module is initialized, refId:%d", tsWal.refId); + wInfo("wal module is initialized, rsetId:%d", tsWal.refId); return code; } @@ -203,14 +203,14 @@ static int32_t walCreateThread() { } pthread_attr_destroy(&thAttr); - wDebug("wal thread is launched"); + wDebug("wal thread is launched, thread:0x%08" PRIx64, taosGetPthreadId(tsWal.thread)); return TSDB_CODE_SUCCESS; } static void walStopThread() { tsWal.stop = 1; - if (tsWal.thread) { + if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index e67127d6e478332889bec3ac565caff62d475b37..4368ddd7d35c444c0b6e32fb5897801bba6e615d 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -39,7 +39,7 @@ int32_t walRenew(void *handle) { if (tfValid(pWal->tfd)) { tfClose(pWal->tfd); - wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name); + wDebug("vgId:%d, file:%s, it is closed while renew", pWal->vgId, pWal->name); } if (pWal->keep == TAOS_WAL_KEEP) { @@ -56,7 +56,7 @@ int32_t walRenew(void *handle) { code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); } else { - wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name); + wDebug("vgId:%d, file:%s, it is created and open while renew", pWal->vgId, pWal->name); } pthread_mutex_unlock(&pWal->mutex); @@ -95,11 +95,15 @@ void walRemoveAllOldFiles(void *handle) { int64_t fileId = -1; pthread_mutex_lock(&pWal->mutex); + + tfClose(pWal->tfd); + wDebug("vgId:%d, file:%s, it is closed before remove all wals", pWal->vgId, pWal->name); + while (walGetNextFile(pWal, &fileId) >= 0) { snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); if (remove(pWal->name) < 0) { - wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name); + wError("vgId:%d, wal:%p file:%s, failed to remove since %s", pWal->vgId, pWal, pWal->name, strerror(errno)); } else { wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name); } @@ -107,6 +111,28 @@ void walRemoveAllOldFiles(void *handle) { pthread_mutex_unlock(&pWal->mutex); } +#if defined(WAL_CHECKSUM_WHOLE) + +static void walUpdateChecksum(SWalHead *pHead) { + pHead->sver = 1; + pHead->cksum = 0; + pHead->cksum = taosCalcChecksum(0, (uint8_t *)pHead, sizeof(*pHead) + pHead->len); +} + +static int walValidateChecksum(SWalHead *pHead) { + if (pHead->sver == 0) { // for compatible with wal before sver 1 + return taosCheckChecksumWhole((uint8_t *)pHead, sizeof(*pHead)); + } else if (pHead->sver == 1) { + uint32_t cksum = pHead->cksum; + pHead->cksum = 0; + return taosCheckChecksum((uint8_t *)pHead, sizeof(*pHead) + pHead->len, cksum); + } + + return 0; +} + +#endif + int32_t walWrite(void *handle, SWalHead *pHead) { if (handle == NULL) return -1; @@ -119,7 +145,13 @@ int32_t walWrite(void *handle, SWalHead *pHead) { if (pHead->version <= pWal->version) return 0; pHead->signature = WAL_SIGNATURE; +#if defined(WAL_CHECKSUM_WHOLE) + walUpdateChecksum(pHead); +#else + pHead->sver = 0; taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead)); +#endif + int32_t contLen = pHead->len + sizeof(SWalHead); pthread_mutex_lock(&pWal->mutex); @@ -192,7 +224,7 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } - wDebug("vgId:%d, file:%s open success", pWal->vgId, pWal->name); + wDebug("vgId:%d, file:%s, it is created and open while restore", pWal->vgId, pWal->name); } return TSDB_CODE_SUCCESS; @@ -242,16 +274,40 @@ static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, continue; } +#if defined(WAL_CHECKSUM_WHOLE) + if (pHead->sver == 0 && walValidateChecksum(pHead)) { + wInfo("vgId:%d, wal head cksum check passed, offset:%" PRId64, pWal->vgId, pos); + *offset = pos; + return TSDB_CODE_SUCCESS; + } + + if (pHead->sver == 1) { + if (tfRead(tfd, pHead->cont, pHead->len) < pHead->len) { + wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); + return TSDB_CODE_WAL_FILE_CORRUPTED; + } + + if (walValidateChecksum(pHead)) { + wInfo("vgId:%d, wal whole cksum check passed, offset:%" PRId64, pWal->vgId, pos); + *offset = pos; + return TSDB_CODE_SUCCESS; + } + } + +#else if (taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { wInfo("vgId:%d, wal head cksum check passed, offset:%" PRId64, pWal->vgId, pos); *offset = pos; return TSDB_CODE_SUCCESS; } + +#endif } return TSDB_CODE_WAL_FILE_CORRUPTED; } + static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, char *name, int64_t fileId) { int32_t size = WAL_MAX_SIZE; void * buffer = tmalloc(size); @@ -265,6 +321,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); tfree(buffer); return TAOS_SYSTEM_ERROR(errno); + } else { + wDebug("vgId:%d, file:%s, open for restore", pWal->vgId, name); } int32_t code = TSDB_CODE_SUCCESS; @@ -272,7 +330,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch SWalHead *pHead = buffer; while (1) { - int32_t ret = tfRead(tfd, pHead, sizeof(SWalHead)); + int32_t ret = (int32_t)tfRead(tfd, pHead, sizeof(SWalHead)); if (ret == 0) break; if (ret < 0) { @@ -287,6 +345,51 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch break; } +#if defined(WAL_CHECKSUM_WHOLE) + if (pHead->sver == 0 && !walValidateChecksum(pHead)) { + wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, tfd, offset); + break; + } + } + + if (pHead->len < 0 || pHead->len > size - sizeof(SWalHead)) { + wError("vgId:%d, file:%s, wal head len out of range, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, tfd, offset); + break; + } + } + + ret = (int32_t)tfRead(tfd, pHead->cont, pHead->len); + if (ret < 0) { + wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); + code = TAOS_SYSTEM_ERROR(errno); + break; + } + + if (ret < pHead->len) { + wError("vgId:%d, file:%s, failed to read wal body, ret:%d len:%d", pWal->vgId, name, ret, pHead->len); + offset += sizeof(SWalHead); + continue; + } + + if (pHead->sver == 1 && !walValidateChecksum(pHead)) { + wError("vgId:%d, file:%s, wal whole cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + pHead->version, pHead->len, offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); + if (code != TSDB_CODE_SUCCESS) { + walFtruncate(pWal, tfd, offset); + break; + } + } + +#else if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, pHead->version, pHead->len, offset); @@ -307,7 +410,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch } } - ret = tfRead(tfd, pHead->cont, pHead->len); + ret = (int32_t)tfRead(tfd, pHead->cont, pHead->len); if (ret < 0) { wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); code = TAOS_SYSTEM_ERROR(errno); @@ -320,6 +423,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch continue; } +#endif offset = offset + sizeof(SWalHead) + pHead->len; wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, @@ -332,6 +436,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch tfClose(tfd); tfree(buffer); + wDebug("vgId:%d, file:%s, it is closed after restore", pWal->vgId, name); return code; } @@ -341,3 +446,16 @@ uint64_t walGetVersion(twalh param) { return pWal->version; } + +// Wal version in slave (dnode1) must be reset. +// Because after the data file is recovered from peer (dnode2), the new file version in dnode1 may become smaller than origin. +// Some new wal record cannot be written to the wal file in dnode1 for wal version not reset, then fversion and the record in wal file may inconsistent, +// At this time, if dnode2 down, dnode1 switched to master. After dnode2 start and restore data from dnode1, data loss will occur + +void walResetVersion(twalh param, uint64_t newVer) { + SWal *pWal = param; + if (pWal == 0) return; + wInfo("vgId:%d, version reset from %" PRIu64 " to %" PRIu64, pWal->vgId, pWal->version, newVer); + + pWal->version = newVer; +} \ No newline at end of file diff --git a/src/wal/test/CMakeLists.txt b/src/wal/test/CMakeLists.txt index b8338b1738a840d43d6f073e5562f0e1ffacf851..f20a57899e049115ded0012c0092bf643af76187 100644 --- a/src/wal/test/CMakeLists.txt +++ b/src/wal/test/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_LINUX) @@ -10,4 +10,12 @@ IF (TD_LINUX) ENDIF () +IF (TD_DARWIN) + INCLUDE_DIRECTORIES(../inc) + + LIST(APPEND WALTEST_SRC ./waltest.c) + ADD_EXECUTABLE(waltest ${WALTEST_SRC}) + TARGET_LINK_LIBRARIES(waltest twal osdetail tutil) + +ENDIF () diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 57fc8b1953d155c6ab45c8adc9e4146d00fb0b39..4e7e9a87ea6810c362bd676cd9152f61bc08e29d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,7 +3,7 @@ # generate release version: # mkdir release; cd release; cmake -DCMAKE_BUILD_TYPE=Release .. -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) SET(CMAKE_C_STANDARD 11) diff --git a/tests/Jenkinsfile b/tests/Jenkinsfile index f5b0cf1478266cc58624c5e6c89cccfb2063b4f6..178a0446c387118b167ee98e0fb39d6466697e92 100644 --- a/tests/Jenkinsfile +++ b/tests/Jenkinsfile @@ -1,9 +1,8 @@ def pre_test(){ - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - sh ''' - sudo rmtaos - ''' - } + + sh ''' + sudo rmtaos||echo 'no taosd installed' + ''' sh ''' cd ${WKC} git reset --hard @@ -56,6 +55,8 @@ pipeline { cd ${WKC}/tests ./test-all.sh b1 date''' + + } } @@ -75,9 +76,20 @@ pipeline { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* ./handle_crash_gen_val_log.sh ''' } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WKC}/tests/pytest + rm -rf /var/lib/taos/* + rm -rf /var/log/taos/* + ./handle_taosd_val_log.sh + ''' + } + sh''' systemctl start taosd sleep 10 @@ -109,6 +121,13 @@ pipeline { java --class-path=../../../../src/connector/jdbc/target:$JAVA_HOME/jre/lib/ext -jar target/JDBCDemo-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 ''' } + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cp -rf ${WKC}/tests/examples/nodejs ${JENKINS_HOME}/workspace/ + cd ${JENKINS_HOME}/workspace/nodejs + node nodejsChecker.js host=localhost + ''' + } catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${JENKINS_HOME}/workspace/C#NET/src/CheckC# @@ -121,6 +140,10 @@ pipeline { ./test-all.sh b2 date ''' + sh ''' + cd ${WKC}/tests + ./test-all.sh full unit + date''' } } @@ -139,6 +162,10 @@ pipeline { ''' } sh ''' + cd ${WKC}/tests + ./test-all.sh full jdbc + date''' + sh ''' cd ${WKC}/tests/pytest ./valgrind-test.sh 2>&1 > mem-error-out.log ./handle_val_log.sh @@ -147,6 +174,11 @@ pipeline { cd ${WKC}/tests ./test-all.sh b3 date''' + sh ''' + date + cd ${WKC}/tests + ./test-all.sh full example + date''' } } diff --git a/tests/comparisonTest/tdengine/CMakeLists.txt b/tests/comparisonTest/tdengine/CMakeLists.txt index aaa18592ed29b6e39c97a55e68e4867684817bbf..36ed3efe191c9d949d6234bd61ffbbe28c3a33d2 100644 --- a/tests/comparisonTest/tdengine/CMakeLists.txt +++ b/tests/comparisonTest/tdengine/CMakeLists.txt @@ -1,7 +1,12 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) IF (TD_LINUX) add_executable(tdengineTest tdengineTest.c) target_link_libraries(tdengineTest taos_static tutil common pthread) ENDIF() + +IF (TD_DARWIN) + add_executable(tdengineTest tdengineTest.c) + target_link_libraries(tdengineTest taos_static tutil common pthread) +ENDIF() diff --git a/tests/examples/C#/taosdemo/taosdemo.cs b/tests/examples/C#/taosdemo/taosdemo.cs index df52acc99dcfeffacbc896d27b81efa66b45fa5f..2d78418e0ab8599f87a544b833b9b6bbd0b48ee7 100644 --- a/tests/examples/C#/taosdemo/taosdemo.cs +++ b/tests/examples/C#/taosdemo/taosdemo.cs @@ -650,6 +650,7 @@ namespace TDengineDriver tester.CloseConnection(); Console.WriteLine("End."); + CleanAndExitProgram(0); } public class InsertDataThread diff --git a/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md b/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md index e14a5f7b675fa1f38514f93afd853e7a1be2a77f..17c5c8df00ab8727d1adfe493d3fbbd32891a676 100644 --- a/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md +++ b/tests/examples/JDBC/JDBCDemo/README-jdbc-windows.md @@ -137,7 +137,7 @@ Welcome to the TDengine shell from Linux, Client Version:2.0.1.1 Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep1,keep2,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | + name | created_time | ntables | vgroups | replica | quorum | days | keep0,keep1,keep(D) | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | precision | status | =================================================================================================================================================================================================================================================================== test | 2020-08-19 18:43:50.731 | 1 | 1 | 1 | 1 | 2 | 3650,3650,3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | ms | ready | log | 2020-08-19 18:40:28.064 | 4 | 1 | 1 | 1 | 10 | 30,30,30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | us | ready | diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index d075fc8f2ad480535075b79efc15c55d9bb799a5..66e866a2d3753b574ba6a4b180fc5740dd4aeef6 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -9,6 +9,14 @@ SNAPSHOT jar + + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.22 + + + @@ -42,18 +50,11 @@ 8 8 + UTF-8 - - - com.taosdata.jdbc - taos-jdbcdriver - 2.0.15 - - - diff --git a/tests/examples/JDBC/JDBCDemo/readme.md b/tests/examples/JDBC/JDBCDemo/readme.md index da638a0bcc485cb3d73f75b59348ec260cc871d2..94844420859a2cc56842e95e4ac8c664c8ffa2e7 100644 --- a/tests/examples/JDBC/JDBCDemo/readme.md +++ b/tests/examples/JDBC/JDBCDemo/readme.md @@ -11,12 +11,12 @@ Download the tdengine package on our website: ``https://www.taosdata.com/cn/all- ## Run jdbcDemo using mvn plugin run command: ``` -mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcDemo" +mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JDBCDemo" ``` and run with your customed args ``` -mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcDemo" -Dexec.args="-host [HOSTNAME]" +mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JDBCDemo" -Dexec.args="-host [HOSTNAME]" ``` ## Compile the Demo Code and Run It diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java index e569de10cf7894aa04fc3cb5bdb8354b581d5a93..da865b3ffde0fbd6af1b39e52a99ca5f190abda6 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java @@ -5,7 +5,6 @@ import java.util.Properties; public class JDBCDemo { private static String host; - private static String driverType = "jni"; private static final String dbName = "test"; private static final String tbName = "weather"; private Connection connection; @@ -14,17 +13,10 @@ public class JDBCDemo { for (int i = 0; i < args.length; i++) { if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) host = args[++i]; - if ("-driverType".equalsIgnoreCase(args[i]) && i < args.length - 1) { - driverType = args[++i]; - if (!"jni".equalsIgnoreCase(driverType) && !"restful".equalsIgnoreCase(driverType)) - printHelp(); - } } - if (host == null) { printHelp(); } - JDBCDemo demo = new JDBCDemo(); demo.init(); demo.createDatabase(); @@ -38,17 +30,11 @@ public class JDBCDemo { } private void init() { + final String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; // get connection try { - String url = "jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"; - if (driverType.equals("restful")) { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; - } else { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - } + Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); - properties.setProperty("host", host); properties.setProperty("charset", "UTF-8"); properties.setProperty("locale", "en_US.UTF-8"); properties.setProperty("timezone", "UTC-8"); @@ -71,11 +57,39 @@ public class JDBCDemo { exuete(sql); } + private void dropTable() { + final String sql = "drop table if exists " + dbName + "." + tbName + ""; + exuete(sql); + } + + private void createTable() { + final String sql = "create table if not exists " + dbName + "." + tbName + " (ts timestamp, temperature float, humidity int)"; + exuete(sql); + } + + private void insert() { + final String sql = "insert into " + dbName + "." + tbName + " (ts, temperature, humidity) values(now, 20.5, 34)"; + exuete(sql); + } + private void select() { - final String sql = "select * from test.weather"; + final String sql = "select * from "+ dbName + "." + tbName; executeQuery(sql); } + private void close() { + try { + if (connection != null) { + this.connection.close(); + System.out.println("connection closed."); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /************************************************************************/ + private void executeQuery(String sql) { try (Statement statement = connection.createStatement()) { long start = System.currentTimeMillis(); @@ -100,15 +114,6 @@ public class JDBCDemo { } } - private void insert() { - final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"; - exuete(sql); - } - - private void createTable() { - final String sql = "create table if not exists " + dbName + "." + tbName + " (ts timestamp, temperature float, humidity int)"; - exuete(sql); - } private void printSql(String sql, boolean succeed, long cost) { System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); @@ -126,24 +131,8 @@ public class JDBCDemo { } } - private void close() { - try { - if (connection != null) { - this.connection.close(); - System.out.println("connection closed."); - } - } catch (SQLException e) { - e.printStackTrace(); - } - } - - private void dropTable() { - final String sql = "drop table if exists " + dbName + "." + tbName + ""; - exuete(sql); - } - private static void printHelp() { - System.out.println("Usage: java -jar JdbcDemo.jar -host -driverType "); + System.out.println("Usage: java -jar JDBCDemo.jar -host "); System.exit(0); } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java new file mode 100644 index 0000000000000000000000000000000000000000..eedb0ba166da50f21577296ad6fa1cd3f838ac32 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JdbcRestfulDemo.java @@ -0,0 +1,47 @@ +package com.taosdata.example; + +import java.sql.*; +import java.util.Properties; + +public class JdbcRestfulDemo { + private static final String host = "master"; + + public static void main(String[] args) { + try { + // load JDBC-restful driver + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + // use port 6041 in url when use JDBC-restful + String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + + Properties properties = new Properties(); + properties.setProperty("charset", "UTF-8"); + properties.setProperty("locale", "en_US.UTF-8"); + properties.setProperty("timezone", "UTC-8"); + + Connection conn = DriverManager.getConnection(url, properties); + Statement stmt = conn.createStatement(); + + stmt.execute("drop database if exists restful_test"); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + stmt.execute("create table restful_test.weather(ts timestamp, temperature float) tags(location nchar(64))"); + stmt.executeUpdate("insert into t1 using restful_test.weather tags('北京') values(now, 18.2)"); + ResultSet rs = stmt.executeQuery("select * from restful_test.weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + + rs.close(); + stmt.close(); + conn.close(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java new file mode 100644 index 0000000000000000000000000000000000000000..def4c649027034028d222bfedb71e37d82b99380 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/SubscribeDemo.java @@ -0,0 +1,75 @@ +package com.taosdata.example; + +import com.taosdata.jdbc.TSDBConnection; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBResultSet; +import com.taosdata.jdbc.TSDBSubscribe; + +import java.sql.DriverManager; +import java.sql.ResultSetMetaData; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class SubscribeDemo { + private static final String usage = "java -jar SubscribeDemo.jar -host -database -topic -sql "; + + public static void main(String[] args) { + // parse args from command line + String host = "", database = "", topic = "", sql = ""; + for (int i = 0; i < args.length; i++) { + if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { + host = args[++i]; + } + if ("-database".equalsIgnoreCase(args[i]) && i < args.length - 1) { + database = args[++i]; + } + if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { + topic = args[++i]; + } + if ("-sql".equalsIgnoreCase(args[i]) && i < args.length - 1) { + sql = args[++i]; + } + } + if (host.isEmpty() || database.isEmpty() || topic.isEmpty() || sql.isEmpty()) { + System.out.println(usage); + return; + } + /*********************************************************************************************/ + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + final String url = "jdbc:TAOS://" + host + ":6030/" + database + "?user=root&password=taosdata"; + // get TSDBConnection + TSDBConnection connection = (TSDBConnection) DriverManager.getConnection(url, properties); + // create TSDBSubscribe + TSDBSubscribe sub = connection.subscribe(topic, sql, false); + + int total = 0; + while (true) { + TSDBResultSet rs = sub.consume(); + int count = 0; + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + count++; + } + total += count; +// System.out.printf("%d rows consumed, total %d\n", count, total); + if (total >= 10) + break; + TimeUnit.SECONDS.sleep(1); + } + sub.close(false); + connection.close(); + } catch (Exception e) { + System.out.println("host: " + host + ", database: " + database + ", topic: " + topic + ", sql: " + sql); + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/tests/examples/JDBC/SpringJdbcTemplate/pom.xml b/tests/examples/JDBC/SpringJdbcTemplate/pom.xml index 15aed1cf03b4e10e5a69e16be99ea2320a24609e..64a91b951bafe65e2f685fb57091221760fb99f9 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/tests/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -47,7 +47,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.18 @@ -69,7 +69,7 @@ - com.taosdata.jdbc.example.jdbcTemplate.App + com.taosdata.example.jdbcTemplate.App diff --git a/tests/examples/JDBC/SpringJdbcTemplate/readme.md b/tests/examples/JDBC/SpringJdbcTemplate/readme.md index 1fe8809b506c248f226edd0f3200c6e352c0a73b..b70a6565f88d0a08b8a26a60676e729ecdb39e2e 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/readme.md +++ b/tests/examples/JDBC/SpringJdbcTemplate/readme.md @@ -8,18 +8,16 @@ 修改 `src/main/resources/applicationContext.xml` 文件中 TDengine 的配置信息: ```xml - - - - - - - - - - - - + + + + + + + + + + ``` ### 打包运行 diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/App.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java similarity index 86% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/App.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java index a03ca3924f9d86199c461a21d887b78e433ef6fe..6942d62a83adafb85496a81ce93866cd0d53611d 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/App.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/App.java @@ -1,9 +1,9 @@ -package com.taosdata.jdbc.example.jdbcTemplate; +package com.taosdata.example.jdbcTemplate; -import com.taosdata.jdbc.example.jdbcTemplate.dao.ExecuteAsStatement; -import com.taosdata.jdbc.example.jdbcTemplate.dao.WeatherDao; -import com.taosdata.jdbc.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.dao.ExecuteAsStatement; +import com.taosdata.example.jdbcTemplate.dao.WeatherDao; +import com.taosdata.example.jdbcTemplate.domain.Weather; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/ExecuteAsStatement.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatement.java similarity index 58% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/ExecuteAsStatement.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatement.java index f146684cc01f75a211182a90e5cfee4058aea413..5947438e408b3b359b138bf989477f60fc327404 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/ExecuteAsStatement.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatement.java @@ -1,4 +1,4 @@ -package com.taosdata.jdbc.example.jdbcTemplate.dao; +package com.taosdata.example.jdbcTemplate.dao; public interface ExecuteAsStatement{ diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/ExecuteAsStatementImpl.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatementImpl.java similarity index 75% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/ExecuteAsStatementImpl.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatementImpl.java index 2700e701cc42034975ff902ebe1da01f040b5a69..059e3dda15ea36bcaad4309a434a06f020d0301b 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/ExecuteAsStatementImpl.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/ExecuteAsStatementImpl.java @@ -1,6 +1,5 @@ -package com.taosdata.jdbc.example.jdbcTemplate.dao.impl; +package com.taosdata.example.jdbcTemplate.dao; -import com.taosdata.jdbc.example.jdbcTemplate.dao.ExecuteAsStatement; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/WeatherDao.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDao.java similarity index 65% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/WeatherDao.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDao.java index 28962ee1e6803898a0ec24e2231a62d91bcbf6d6..19a07597f8df869dada4e258c4951f67132107a0 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/WeatherDao.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDao.java @@ -1,6 +1,6 @@ -package com.taosdata.jdbc.example.jdbcTemplate.dao; +package com.taosdata.example.jdbcTemplate.dao; -import com.taosdata.jdbc.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.domain.Weather; import java.util.List; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/WeatherDaoImpl.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDaoImpl.java similarity index 84% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/WeatherDaoImpl.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDaoImpl.java index 1e0e0ab68c63bdb87ad172f277fe2a65df79d229..8d4ca47d5ee5d351a48bcc1554531566116f5a52 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/dao/impl/WeatherDaoImpl.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/dao/WeatherDaoImpl.java @@ -1,20 +1,16 @@ -package com.taosdata.jdbc.example.jdbcTemplate.dao.impl; +package com.taosdata.example.jdbcTemplate.dao; -import com.taosdata.jdbc.example.jdbcTemplate.dao.WeatherDao; -import com.taosdata.jdbc.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.dao.WeatherDao; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.namedparam.SqlParameterSourceUtils; -import org.springframework.jdbc.core.simple.SimpleJdbcInsert; import org.springframework.stereotype.Repository; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Timestamp; -import java.util.HashMap; import java.util.List; -import java.util.Map; @Repository public class WeatherDaoImpl implements WeatherDao { diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/domain/Weather.java b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/domain/Weather.java similarity index 94% rename from tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/domain/Weather.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/domain/Weather.java index 023b301481829b8d73a8af566d418aa8232ef899..1787a08c3554e017645fefbd35d7c4fd1bc60bd4 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/jdbc/example/jdbcTemplate/domain/Weather.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/java/com/taosdata/example/jdbcTemplate/domain/Weather.java @@ -1,4 +1,4 @@ -package com.taosdata.jdbc.example.jdbcTemplate.domain; +package com.taosdata.example.jdbcTemplate.domain; import java.sql.Timestamp; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/main/resources/applicationContext.xml b/tests/examples/JDBC/SpringJdbcTemplate/src/main/resources/applicationContext.xml index 19ac433385fabe53548dcba25d5bb10965df17af..6d6cf6047e2545256ffd6f39e1e6160a965a6fa8 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/main/resources/applicationContext.xml +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/main/resources/applicationContext.xml @@ -10,7 +10,7 @@ - + @@ -20,6 +20,6 @@ - + diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/example/jdbcTemplate/BatcherInsertTest.java b/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java similarity index 89% rename from tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/example/jdbcTemplate/BatcherInsertTest.java rename to tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java index 2f2446eb7049ae44ff3e142d62a2e51ab71b6f16..29d0f79fd4982d43078e590b4320c0df457ee44c 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/example/jdbcTemplate/BatcherInsertTest.java +++ b/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/example/jdbcTemplate/BatcherInsertTest.java @@ -1,9 +1,9 @@ -package com.taosdata.jdbc.example.jdbcTemplate; +package com.taosdata.example.jdbcTemplate; -import com.taosdata.jdbc.example.jdbcTemplate.dao.ExecuteAsStatement; -import com.taosdata.jdbc.example.jdbcTemplate.dao.WeatherDao; -import com.taosdata.jdbc.example.jdbcTemplate.domain.Weather; +import com.taosdata.example.jdbcTemplate.dao.ExecuteAsStatement; +import com.taosdata.example.jdbcTemplate.dao.WeatherDao; +import com.taosdata.example.jdbcTemplate.domain.Weather; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/AppTest.java b/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/AppTest.java deleted file mode 100644 index d0219f3db73d0c260420b642f997663acdd36f87..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/SpringJdbcTemplate/src/test/java/com/taosdata/jdbc/AppTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.taosdata.jdbc; - -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -/** - * Unit test for simple App. - */ -public class AppTest { - /** - * Rigorous Test :-) - */ - @Test - public void shouldAnswerWithTrue() { - assertTrue(true); - } -} diff --git a/tests/examples/JDBC/calciteDemo/pom.xml b/tests/examples/JDBC/calciteDemo/pom.xml deleted file mode 100644 index a7d47837d9ba407f3959a00d522a9cd26b5622db..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/calciteDemo/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - 4.0.0 - - com.taosdata.example.calcite - calciteDemo - 1.0-SNAPSHOT - - - - - org.slf4j - slf4j-simple - 1.7.25 - compile - - - - org.apache.calcite - calcite-core - 1.23.0 - - - org.apache.commons - commons-dbcp2 - 2.7.0 - - - org.apache.calcite.avatica - avatica-core - 1.17.0 - - - - - mysql - mysql-connector-java - 5.1.47 - - - - - com.taosdata.jdbc - taos-jdbcdriver - 2.0.8 - - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java b/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java deleted file mode 100644 index 7e97956b7879c26056b9b0e27cce4786284ff96c..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/calciteDemo/src/main/java/com/taosdata/example/calcite/CalciteDemo.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.taosdata.example.calcite; - -import org.apache.calcite.adapter.jdbc.JdbcSchema; -import org.apache.calcite.jdbc.CalciteConnection; -import org.apache.calcite.schema.Schema; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.sql.parser.SqlParseException; -import org.apache.commons.dbcp2.BasicDataSource; - -import java.sql.*; -import java.util.Properties; - -public class CalciteDemo { - - private static String url_taos = "jdbc:TAOS://192.168.236.135:6030/test"; - private static String url_mysql = "jdbc:mysql://master:3306/test?useSSL=false&useUnicode=true&characterEncoding=UTF-8"; - - public static void main(String[] args) throws SqlParseException, ClassNotFoundException, SQLException { - Class.forName("org.apache.calcite.jdbc.Driver"); - Properties info = new Properties(); - info.setProperty("caseSensitive", "false"); - - Connection connection = DriverManager.getConnection("jdbc:calcite:", info); - CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); - - SchemaPlus rootSchema = calciteConnection.getRootSchema(); - - //这里hdb是在tdengine中创建的数据库名 - Schema schema = mysqlTest(rootSchema); -// Schema schema = tdengineTest(rootSchema); - - //创建新的schema自动映射到原来的hdb数据库 - rootSchema.add("test", schema); - - Statement stmt = calciteConnection.createStatement(); - //查询schema test中的表,表名是tdengine中的表 - ResultSet rs = stmt.executeQuery("select * from test.t"); - ResultSetMetaData metaData = rs.getMetaData(); - while (rs.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - System.out.println(metaData.getColumnLabel(i) + " : " + rs.getString(i)); - } - } - } - - - private static Schema tdengineTest(SchemaPlus rootSchema) throws ClassNotFoundException { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(url_taos); - dataSource.setUsername("root"); - dataSource.setPassword("taosdata"); - - return JdbcSchema.create(rootSchema, "test", dataSource, "hdb", null); - } - - private static Schema mysqlTest(SchemaPlus rootSchema) throws ClassNotFoundException { - Class.forName("com.mysql.jdbc.Driver"); - BasicDataSource dataSource = new BasicDataSource(); - dataSource.setUrl(url_mysql); - dataSource.setUsername("root"); - dataSource.setPassword("123456"); - - //Schema schema = JdbcSchema.create(rootSchema, "test", dataSource, "hdb", null); - return JdbcSchema.create(rootSchema, "test", dataSource, "test", null); - } -} diff --git a/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties b/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties deleted file mode 100644 index 1a77ec520cd3e727ac51acb4b8f647b7cf188d97..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/calciteDemo/src/main/resources/log4j.properties +++ /dev/null @@ -1,6 +0,0 @@ -log4j.rootLogger=info,stdout - -#console -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern= [%d{yyyy-MM-dd HH:mm:ss a}]:%p %l%m%n \ No newline at end of file diff --git a/tests/examples/JDBC/connectionPools/README-cn.md b/tests/examples/JDBC/connectionPools/README-cn.md index 761596dfc55a3e2c9f449ed34fd72ac96c277512..9b26df3c2eb2c23171a673643891a292af4c920c 100644 --- a/tests/examples/JDBC/connectionPools/README-cn.md +++ b/tests/examples/JDBC/connectionPools/README-cn.md @@ -1,23 +1,25 @@ 这个example中,我们适配了java常见的连接池: -* c3p0 -* dbcp +* HikariCP(默认) * druid -* HikariCP +* dbcp +* c3p0 ### 说明 ConnectionPoolDemo的程序逻辑: 1. 创建到host的connection连接池 2. 创建名称为pool_test的database,创建表超级weather,创建tableSize个子表 -3. 不断向所有子表进行插入。 +3. 总共插入totalNumber条数据。 ### 如何运行这个例子: + ```shell script -# mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.args="-host localhost" +mvn clean package assembly:single +java -jar target/connectionPools-1.0-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 ``` 使用mvn运行ConnectionPoolDemo的main方法,可以指定参数 ```shell script Usage: -mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.args="" +java -jar target/connectionPools-1.0-SNAPSHOT-jar-with-dependencies.jar -host : hostname -poolType -poolSize @@ -26,8 +28,5 @@ mvn exec:java -Dexec.mainClass="com.taosdata.demo.ConnectionPoolDemo" -Dexec.arg -sleep : 每次插入任务提交后的 ``` -### 如何停止程序: -ConnectionPoolDemo不会自己停止,会一直执行插入,需要手动Ctrl+C运行。 - ### 日志 使用log4j,将日志和错误分别输出到了debug.log和error.log中 \ No newline at end of file diff --git a/tests/examples/JDBC/connectionPools/pom.xml b/tests/examples/JDBC/connectionPools/pom.xml index 2793f0a83ddc88711796c133802c82979ae14be5..84467003f905c454bb285088e56bcd26465a3e5e 100644 --- a/tests/examples/JDBC/connectionPools/pom.xml +++ b/tests/examples/JDBC/connectionPools/pom.xml @@ -12,7 +12,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.11 + 2.0.18 @@ -44,13 +44,41 @@ c3p0 0.9.5.4 - log4j log4j 1.2.17 - + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.1.0 + + + + com.taosdata.example.ConnectionPoolDemo + + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java deleted file mode 100644 index ed86acd6e9f8bfb8c862c1764e39f541d3f054eb..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/common/InsertTask.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.taosdata.demo.common; - -import org.apache.log4j.Logger; - -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Random; - -public class InsertTask implements Runnable { - private final Random random = new Random(System.currentTimeMillis()); - private static final Logger logger = Logger.getLogger(InsertTask.class); - - private final DataSource ds; - private final int batchSize; - private final String dbName; - private final int tableSize; - - public InsertTask(DataSource ds, String dbName, int tableSize, int batchSize) { - this.ds = ds; - this.dbName = dbName; - this.tableSize = tableSize; - this.batchSize = batchSize; - } - - @Override - public void run() { - Connection conn = null; - Statement stmt = null; - int affectedRows = 0; - - long start = System.currentTimeMillis(); - try { - conn = ds.getConnection(); - stmt = conn.createStatement(); - - for (int tb_index = 1; tb_index <= tableSize; tb_index++) { - StringBuilder sb = new StringBuilder(); - sb.append("insert into "); - sb.append(dbName); - sb.append(".t_"); - sb.append(tb_index); - sb.append("(ts, temperature, humidity) values "); - for (int i = 0; i < batchSize; i++) { - sb.append("("); - sb.append(start + i); - sb.append(", "); - sb.append(random.nextFloat() * 30); - sb.append(", "); - sb.append(random.nextInt(70)); - sb.append(") "); - } - logger.info("SQL >>> " + sb.toString()); - affectedRows += stmt.executeUpdate(sb.toString()); - } - } catch (SQLException e) { - e.printStackTrace(); - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - logger.info(">>> affectedRows:" + affectedRows + " TimeCost:" + (System.currentTimeMillis() - start) + " ms"); - } - } -} diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java deleted file mode 100644 index 87f1f4ad2cbba41a779f0247f2214ef2bf04a8ca..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/HikariCpBuilder.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.taosdata.demo.pool; - -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; - -import javax.sql.DataSource; - -public class HikariCpBuilder { - - public static DataSource getDataSource(String host, int poolSize) { - HikariConfig config = new HikariConfig(); - config.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); - config.setJdbcUrl("jdbc:TAOS://" + host + ":6030"); - config.setUsername("root"); - config.setPassword("taosdata"); - - config.setMaximumPoolSize(poolSize); - config.setMinimumIdle(poolSize); - HikariDataSource ds = new HikariDataSource(config); - return ds; - } -} diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java similarity index 54% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java index 4e33b75bc57aa18ebc0cae166e84401d4357d614..bd57d138b21034f45569ab3dcfc8e1ad5b39263d 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java @@ -1,10 +1,10 @@ -package com.taosdata.demo; +package com.taosdata.example; -import com.taosdata.demo.common.InsertTask; -import com.taosdata.demo.pool.C3p0Builder; -import com.taosdata.demo.pool.DbcpBuilder; -import com.taosdata.demo.pool.DruidPoolBuilder; -import com.taosdata.demo.pool.HikariCpBuilder; +import com.taosdata.example.common.InsertTask; +import com.taosdata.example.pool.C3p0Builder; +import com.taosdata.example.pool.DbcpBuilder; +import com.taosdata.example.pool.DruidPoolBuilder; +import com.taosdata.example.pool.HikariCpBuilder; import org.apache.log4j.Logger; import javax.sql.DataSource; @@ -20,44 +20,43 @@ public class ConnectionPoolDemo { private static Logger logger = Logger.getLogger(DruidPoolBuilder.class); private static final String dbName = "pool_test"; - private static int batchSize = 10; - private static int sleep = 1000; - private static int poolSize = 50; - private static int tableSize = 1000; - private static int threadCount = 50; private static String poolType = "hikari"; + private static long totalSize = 1_000_000l; + private static long tableSize = 1; + private static long batchSize = 1; + private static int poolSize = 50; + private static int threadCount = 50; + private static int sleep = 0; - public static void main(String[] args) throws InterruptedException { + public static void main(String[] args) { String host = null; for (int i = 0; i < args.length; i++) { if ("-host".equalsIgnoreCase(args[i]) && i < args.length - 1) { host = args[++i]; } - if ("-batchSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - batchSize = Integer.parseInt(args[++i]); - } - if ("-sleep".equalsIgnoreCase(args[i]) && i < args.length - 1) { - sleep = Integer.parseInt(args[++i]); + if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) { + poolType = args[++i]; } - if ("-poolSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - poolSize = Integer.parseInt(args[++i]); + if ("-recordNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + totalSize = Long.parseLong(args[++i]); } - if ("-tableSize".equalsIgnoreCase(args[i]) && i < args.length - 1) { - tableSize = Integer.parseInt(args[++i]); + if ("-tableNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + tableSize = Long.parseLong(args[++i]); } - if ("-poolType".equalsIgnoreCase(args[i]) && i < args.length - 1) { - poolType = args[++i]; + if ("-batchNumber".equalsIgnoreCase(args[i]) && i < args.length - 1) { + batchSize = Long.parseLong(args[++i]); } + } if (host == null) { - System.out.println("Usage: java -jar XXX.jar " + - "-host " + - "-batchSize " + - "-sleep " + - "-poolSize " + - "-tableSize " + - "-poolType "); + System.out.println("Usage: java -jar XXX.jar -host " + + "-poolType " + + "-recordNumber " + + "-tableNumber " + + "-batchNumber " + + "-sleep " + ); return; } @@ -79,30 +78,35 @@ public class ConnectionPoolDemo { } logger.info(">>>>>>>>>>>>>> connection pool Type: " + poolType); - init(dataSource); - try { - Connection connection = dataSource.getConnection(); - Statement statement = connection.createStatement(); - String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; - int affectRows = statement.executeUpdate(sql); - System.out.println("affectRows >>> " + affectRows); - affectRows = statement.executeUpdate(sql); - System.out.println("affectRows >>> " + affectRows); - statement.close(); - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } +// try { +// Connection connection = dataSource.getConnection(); +// Statement statement = connection.createStatement(); +// String sql = "insert into " + dbName + ".t_1 values('2020-01-01 00:00:00.000',12.12,111)"; +// int affectRows = statement.executeUpdate(sql); +// System.out.println("affectRows >>> " + affectRows); +// affectRows = statement.executeUpdate(sql); +// System.out.println("affectRows >>> " + affectRows); +// statement.close(); +// connection.close(); +// } catch (SQLException e) { +// e.printStackTrace(); +// } + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + for (long i = 0; i < totalSize / tableSize / batchSize; i++) { + executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); + // sleep few seconds + try { + if (sleep > 0) + TimeUnit.MILLISECONDS.sleep(sleep); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + executor.shutdown(); -// ExecutorService executor = Executors.newFixedThreadPool(threadCount); -// while (true) { -// executor.execute(new InsertTask(dataSource, dbName, tableSize, batchSize)); -// if (sleep > 0) -// TimeUnit.MILLISECONDS.sleep(sleep); -// } } private static void init(DataSource dataSource) { diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java new file mode 100644 index 0000000000000000000000000000000000000000..da7c9a22b5b3e7f5d877a3a1489d55f439bff883 --- /dev/null +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java @@ -0,0 +1,46 @@ +package com.taosdata.example.common; + +import org.apache.log4j.Logger; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Random; + +public class InsertTask implements Runnable { + private final Random random = new Random(System.currentTimeMillis()); + private static final Logger logger = Logger.getLogger(InsertTask.class); + + private final DataSource ds; + private final String dbName; + private final long tableSize; + private final long batchSize; + + public InsertTask(DataSource ds, String dbName, long tableSize, long batchSize) { + this.ds = ds; + this.dbName = dbName; + this.tableSize = tableSize; + this.batchSize = batchSize; + } + + @Override + public void run() { + int affectedRows = 0; + long start = System.currentTimeMillis(); + try (Connection conn = ds.getConnection(); Statement stmt = conn.createStatement()) { + for (int tb_index = 1; tb_index <= tableSize; tb_index++) { + StringBuilder sb = new StringBuilder(); + sb.append("insert into ").append(dbName).append(".t_").append(tb_index).append("(ts, temperature, humidity) values "); + for (int i = 0; i < batchSize; i++) { + sb.append("(").append(start + i).append(", ").append(random.nextFloat() * 30).append(", ").append(random.nextInt(70)).append(") "); + } + logger.info("SQL >>> " + sb.toString()); + affectedRows += stmt.executeUpdate(sb.toString()); + } + } catch (SQLException e) { + e.printStackTrace(); + } + logger.info(">>> affectedRows:" + affectedRows + " TimeCost:" + (System.currentTimeMillis() - start) + " ms"); + } +} diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java similarity index 89% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java index 587f417410f96f43be2ced5a4820cd49cdb99a17..235db0bb796ee507b2d93838c8e7bfa7de870fcd 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/C3p0Builder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/C3p0Builder.java @@ -1,7 +1,6 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import com.mchange.v2.c3p0.ComboPooledDataSource; -import org.apache.commons.dbcp.BasicDataSource; import javax.sql.DataSource; import java.beans.PropertyVetoException; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java similarity index 93% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java index 3c34a32532f595bf3134942094e96e952bd09dbb..3aa9e4ebcf2d07ba1e6a51a2dae216cf44e88486 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DbcpBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DbcpBuilder.java @@ -1,4 +1,4 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import org.apache.commons.dbcp.BasicDataSource; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java similarity index 74% rename from tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java rename to tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java index e5dc14c6a5ef69c2a7059d5d78b621e25ff3d799..500f0e9e97cc09c2c5e9990234f051d0bbc72044 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/demo/pool/DruidPoolBuilder.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/DruidPoolBuilder.java @@ -1,4 +1,4 @@ -package com.taosdata.demo.pool; +package com.taosdata.example.pool; import com.alibaba.druid.pool.DruidDataSource; @@ -10,22 +10,18 @@ public class DruidPoolBuilder { final String url = "jdbc:TAOS://" + host + ":6030"; DruidDataSource dataSource = new DruidDataSource(); - dataSource.setUrl(url); + // jdbc properties dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + dataSource.setUrl(url); dataSource.setUsername("root"); dataSource.setPassword("taosdata"); - - //初始连接数,默认0 + // pool configurations dataSource.setInitialSize(poolSize); - //最大连接数,默认8 - dataSource.setMaxActive(poolSize); - //最小闲置数 dataSource.setMinIdle(poolSize); - //获取连接的最大等待时间,单位毫秒 - dataSource.setMaxWait(2000); - + dataSource.setMaxActive(poolSize); + dataSource.setMaxWait(30000); + dataSource.setValidationQuery("select server_status()"); return dataSource; } - } diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java new file mode 100644 index 0000000000000000000000000000000000000000..7e151de3e052495b3ce50ffe042732717fcc38f4 --- /dev/null +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/pool/HikariCpBuilder.java @@ -0,0 +1,28 @@ +package com.taosdata.example.pool; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +import javax.sql.DataSource; + +public class HikariCpBuilder { + + public static DataSource getDataSource(String host, int poolSize) { + HikariConfig config = new HikariConfig(); + // jdbc properties + config.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + config.setJdbcUrl("jdbc:TAOS://" + host + ":6030"); + config.setUsername("root"); + config.setPassword("taosdata"); + // pool configurations + config.setMinimumIdle(poolSize); //minimum number of idle connection + config.setMaximumPoolSize(poolSize); //maximum number of connection in the pool + config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setMaxLifetime(0); // maximum life time for each connection + config.setIdleTimeout(0); // max idle time for recycle idle connection + config.setConnectionTestQuery("select server_status()"); //validation query + + HikariDataSource ds = new HikariDataSource(config); + return ds; + } +} diff --git a/tests/examples/JDBC/mybatisplus-demo/pom.xml b/tests/examples/JDBC/mybatisplus-demo/pom.xml index e59b915d2cba0c53bfcea36782ba25f4fa19c37b..a83d0a00e69bdcffff2db8b17c763959dc67365b 100644 --- a/tests/examples/JDBC/mybatisplus-demo/pom.xml +++ b/tests/examples/JDBC/mybatisplus-demo/pom.xml @@ -47,15 +47,9 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.14 + 2.0.18 - - - mysql - mysql-connector-java - 5.1.47 - org.springframework.boot spring-boot-starter-web diff --git a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml index 71e518602e8f68b9f6691cba60cb6ad0de247bfe..38180c6d75a620a63bcaab9ec350d97e65f9dd16 100644 --- a/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml +++ b/tests/examples/JDBC/mybatisplus-demo/src/main/resources/application.yml @@ -1,19 +1,15 @@ spring: datasource: driver-class-name: com.taosdata.jdbc.TSDBDriver - url: jdbc:TAOS://localhost:6030/mp_test + url: jdbc:TAOS://localhost:6030/mp_test?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8 user: root password: taosdata - charset: UTF-8 - locale: en_US.UTF-8 - timezone: UTC-8 druid: initial-size: 5 min-idle: 5 max-active: 5 - mybatis-plus: configuration: map-underscore-to-camel-case: false diff --git a/tests/examples/JDBC/readme.md b/tests/examples/JDBC/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..9a017f4feab148cb7c3fd4132360c3075c6573cb --- /dev/null +++ b/tests/examples/JDBC/readme.md @@ -0,0 +1,13 @@ +# TDengine examples + +| No. | Name | Describe | +| :--: | :----------------: | ------------------------------------------------------------ | +| 1 | JDBCDemo | Example codes for JDBC-JNI, JDBC-RESTful, Subscribe | +| 2 | connectionPools | Example codes for HikariCP, Druid, dbcp, c3p0 connection pools | +| 3 | SpringJdbcTemplate | Example codes for spring jdbcTemplate | +| 4 | mybatisplus-demo | Example codes for mybatis | +| 5 | springbootdemo | Example codes for springboot | +| 6 | taosdemo | This is an internal tool for testing Our JDBC-JNI, JDBC-RESTful, RESTful interfaces | + + +more detail: https://www.taosdata.com/cn//documentation20/connector-java/ \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/pom.xml b/tests/examples/JDBC/springbootdemo/pom.xml index 881ea0d6bf485891a29427853bd1bbf9b294c2d8..bd5f7efbc0321962a3f3f41a17bc9bde5dee9b27 100644 --- a/tests/examples/JDBC/springbootdemo/pom.xml +++ b/tests/examples/JDBC/springbootdemo/pom.xml @@ -8,7 +8,7 @@ 2.2.1.RELEASE - com.taosdata.jdbc + com.taosdata.example springbootdemo 0.0.1-SNAPSHOT springbootdemo @@ -63,7 +63,9 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.4 + 2.0.20 + + @@ -71,8 +73,6 @@ druid-spring-boot-starter 1.1.17 - - diff --git a/tests/examples/JDBC/springbootdemo/readme.md b/tests/examples/JDBC/springbootdemo/readme.md index ac3bb44ef95c251e25c664cd50414494e19f6820..67a28947d2dfb8fc069bf94fd139a7006d35a22b 100644 --- a/tests/examples/JDBC/springbootdemo/readme.md +++ b/tests/examples/JDBC/springbootdemo/readme.md @@ -47,7 +47,7 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug * 插入单条记录 ```xml - + insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) ``` @@ -67,9 +67,9 @@ logging.level.com.taosdata.jdbc.springbootdemo.dao=debug - + - + diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java similarity index 66% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java index f693214567fb3e8780e8ee1a867cf765de0ec8d7..8f30c299466ca1f83bc689928b08e8001a87908d 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplication.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/SpringbootdemoApplication.java @@ -1,15 +1,13 @@ -package com.taosdata.jdbc.springbootdemo; +package com.taosdata.example.springbootdemo; import org.mybatis.spring.annotation.MapperScan; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@MapperScan(basePackages = {"com.taosdata.jdbc.springbootdemo.dao"}) +@MapperScan(basePackages = {"com.taosdata.example.springbootdemo.dao"}) @SpringBootApplication -public class cd { - +public class SpringbootdemoApplication { public static void main(String[] args) { SpringApplication.run(SpringbootdemoApplication.class, args); } - -} +} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java similarity index 60% rename from tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java rename to tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java index 56a58fbb4d9a783a34989eae00369ff7b307acaf..c153e27701403b672709a3585603b8630796d178 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/controller/WeatherController.java @@ -1,11 +1,12 @@ -package com.taosdata.jdbc.springbootdemo.controller; +package com.taosdata.example.springbootdemo.controller; -import com.taosdata.jdbc.springbootdemo.domain.Weather; -import com.taosdata.jdbc.springbootdemo.service.WeatherService; +import com.taosdata.example.springbootdemo.domain.Weather; +import com.taosdata.example.springbootdemo.service.WeatherService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; import java.util.List; +import java.util.Map; @RequestMapping("/weather") @RestController @@ -20,7 +21,7 @@ public class WeatherController { * @return */ @GetMapping("/init") - public boolean init() { + public int init() { return weatherService.init(); } @@ -44,21 +45,23 @@ public class WeatherController { * @return */ @PostMapping("/{temperature}/{humidity}") - public int saveWeather(@PathVariable int temperature, @PathVariable float humidity) { - + public int saveWeather(@PathVariable float temperature, @PathVariable int humidity) { return weatherService.save(temperature, humidity); } - /** - * upload multi weather info - * - * @param weatherList - * @return - */ - @PostMapping("/batch") - public int batchSaveWeather(@RequestBody List weatherList) { + @GetMapping("/count") + public int count() { + return weatherService.count(); + } + + @GetMapping("/subTables") + public List getSubTables() { + return weatherService.getSubTables(); + } - return weatherService.save(weatherList); + @GetMapping("/avg") + public List avg() { + return weatherService.avg(); } } diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..ad6733558a9d548be196cf8c9c0c63dc96227b39 --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.java @@ -0,0 +1,29 @@ +package com.taosdata.example.springbootdemo.dao; + +import com.taosdata.example.springbootdemo.domain.Weather; +import org.apache.ibatis.annotations.Param; + +import java.util.List; +import java.util.Map; + +public interface WeatherMapper { + + void dropDB(); + + void createDB(); + + void createSuperTable(); + + void createTable(Weather weather); + + List select(@Param("limit") Long limit, @Param("offset") Long offset); + + int insert(Weather weather); + + int count(); + + List getSubTables(); + + List avg(); + +} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml new file mode 100644 index 0000000000000000000000000000000000000000..2d3e0540650f35c1018992795ac33fb6cb7c4837 --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/dao/WeatherMapper.xml @@ -0,0 +1,60 @@ + + + + + + + + + + + + + drop database if exists test + + + + create database if not exists test + + + + create table if not exists test.weather(ts timestamp, temperature float, humidity float) tags(location nchar(64), groupId int) + + + + create table if not exists test.t#{groupId} using test.weather tags(#{location}, #{groupId}) + + + + + + insert into test.t#{groupId} (ts, temperature, humidity) values (#{ts}, ${temperature}, ${humidity}) + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java new file mode 100644 index 0000000000000000000000000000000000000000..255b2cdca920a5568338e823d0744f2e9c4db7d3 --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/domain/Weather.java @@ -0,0 +1,64 @@ +package com.taosdata.example.springbootdemo.domain; + +import com.fasterxml.jackson.annotation.JsonFormat; + +import java.sql.Timestamp; + +public class Weather { + + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS", timezone = "GMT+8") + private Timestamp ts; + private float temperature; + private float humidity; + private String location; + private int groupId; + + public Weather() { + } + + public Weather(Timestamp ts, float temperature, float humidity) { + this.ts = ts; + this.temperature = temperature; + this.humidity = humidity; + } + + public Timestamp getTs() { + return ts; + } + + public void setTs(Timestamp ts) { + this.ts = ts; + } + + public float getTemperature() { + return temperature; + } + + public void setTemperature(float temperature) { + this.temperature = temperature; + } + + public float getHumidity() { + return humidity; + } + + public void setHumidity(float humidity) { + this.humidity = humidity; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public int getGroupId() { + return groupId; + } + + public void setGroupId(int groupId) { + this.groupId = groupId; + } +} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java new file mode 100644 index 0000000000000000000000000000000000000000..0aef828e1cf19f9c612f9c8d0433ce1a361c7441 --- /dev/null +++ b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/example/springbootdemo/service/WeatherService.java @@ -0,0 +1,61 @@ +package com.taosdata.example.springbootdemo.service; + +import com.taosdata.example.springbootdemo.dao.WeatherMapper; +import com.taosdata.example.springbootdemo.domain.Weather; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.sql.Timestamp; +import java.util.List; +import java.util.Map; +import java.util.Random; + +@Service +public class WeatherService { + + @Autowired + private WeatherMapper weatherMapper; + private Random random = new Random(System.currentTimeMillis()); + private String[] locations = {"北京", "上海", "广州", "深圳", "天津"}; + + public int init() { + weatherMapper.dropDB(); + weatherMapper.createDB(); + weatherMapper.createSuperTable(); + long ts = System.currentTimeMillis(); + long thirtySec = 1000 * 30; + int count = 0; + for (int i = 0; i < 20; i++) { + Weather weather = new Weather(new Timestamp(ts + (thirtySec * i)), 30 * random.nextFloat(), random.nextInt(100)); + weather.setLocation(locations[random.nextInt(locations.length)]); + weather.setGroupId(i % locations.length); + weatherMapper.createTable(weather); + count += weatherMapper.insert(weather); + } + return count; + } + + public List query(Long limit, Long offset) { + return weatherMapper.select(limit, offset); + } + + public int save(float temperature, int humidity) { + Weather weather = new Weather(); + weather.setTemperature(temperature); + weather.setHumidity(humidity); + + return weatherMapper.insert(weather); + } + + public int count() { + return weatherMapper.count(); + } + + public List getSubTables() { + return weatherMapper.getSubTables(); + } + + public List avg() { + return weatherMapper.avg(); + } +} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java deleted file mode 100644 index 844ac21bb8855de06edcc586deb713bcad686385..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.controller; - - -import com.taosdata.jdbc.springbootdemo.domain.Rainfall; -import com.taosdata.jdbc.springbootdemo.service.RainStationService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -@RestController -@RequestMapping("/rainstation") -public class RainStationController { - - @Autowired - private RainStationService service; - - @GetMapping("/init") - public boolean init() { - service.init(); - service.createTable(); - return true; - } - - @PostMapping("/insert") - public int insert(@RequestBody Rainfall rainfall){ - return service.insert(rainfall); - } - -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java deleted file mode 100644 index a9266acb3028c453062ce6e56ccade53c405f742..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import java.util.Map; - -public interface DatabaseMapper { - - int createDatabase(String dbname); - - int dropDatabase(String dbname); - - int creatDatabaseWithParameters(Map map); - - int useDatabase(String dbname); - -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml deleted file mode 100644 index 329f75b582a276bed522e671d0c9f868b837c172..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - create database if not exists ${dbname} - - - - DROP database if exists ${dbname} - - - - - CREATE database if not EXISTS ${dbname} - - KEEP ${keep} - - - DAYS ${days} - - - REPLICA ${replica} - - - cache ${cache} - - - blocks ${blocks} - - - minrows ${minrows} - - - maxrows ${maxrows} - - - - - use ${dbname} - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java deleted file mode 100644 index f0efbf40ba477d9b8f410e0806be38701d274394..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import java.util.Map; - -public interface RainfallMapper { - - - int save(Map map); -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml deleted file mode 100644 index 319b4f397434f1b7623c31250c8ba798c0499d86..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - INSERT INTO ${table} using ${dbname}.${stable} tags(#{values.station_code}, #{values.station_name}) (ts, name, code, rainfall) values (#{values.ts}, #{values.name}, #{values.code}, #{values.rainfall}) - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java deleted file mode 100644 index 7601bf974c7c727b2a46171b145bdcec662dc258..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import com.taosdata.jdbc.springbootdemo.domain.TableMetadata; - -public interface TableMapper { - - boolean createSTable(TableMetadata tableMetadata); -} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml deleted file mode 100644 index 5a272eadb4abd7637735a834ec445c822073486f..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - create table if not exists ${dbname}.${tablename} - - ${field.name} ${field.type} - - TAGS - - ${tag.name} ${tag.type} - - - - - drop ${tablename} - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java deleted file mode 100644 index 1e3db1f49106606c412851c0a74ad382adea68fb..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.dao; - -import com.taosdata.jdbc.springbootdemo.domain.Weather; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -public interface WeatherMapper { - - int insert(Weather weather); - - int batchInsert(List weatherList); - - List select(@Param("limit") Long limit, @Param("offset")Long offset); - - void createDB(); - - void createTable(); -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml deleted file mode 100644 index e894f9a6583d271d8ce526e9afe79528f0fd5490..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/WeatherMapper.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - create database if not exists test; - - - - create table if not exists test.weather(ts timestamp, temperature int, humidity float); - - - - ts, temperature, humidity - - - - - - insert into test.weather (ts, temperature, humidity) values (now, #{temperature,jdbcType=INTEGER}, #{humidity,jdbcType=FLOAT}) - - - - insert into test.weather (ts, temperature, humidity) values - - (now + #{index}a, #{weather.temperature}, #{weather.humidity}) - - - - - \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java deleted file mode 100644 index 619b5a303d715132d6a428770a27d069a9b9dcaa..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -public class FieldMetadata { - - private String name; - private String type; - - public FieldMetadata(String name, String type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java deleted file mode 100644 index 93e199d7e66582e7f8d771994b358a2c58f5dae2..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -import com.fasterxml.jackson.annotation.JsonFormat; - -import java.sql.Timestamp; - -public class Rainfall { - - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS",timezone = "GMT+8") - private Timestamp ts; - private String name; - private String code; - private float rainfall; - private String station_code; - private String station_name; - - public Timestamp getTs() { - return ts; - } - - public void setTs(Timestamp ts) { - this.ts = ts; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getCode() { - return code; - } - - public void setCode(String code) { - this.code = code; - } - - public float getRainfall() { - return rainfall; - } - - public void setRainfall(float rainfall) { - this.rainfall = rainfall; - } - - public String getStation_code() { - return station_code; - } - - public void setStation_code(String station_code) { - this.station_code = station_code; - } - - public String getStation_name() { - return station_name; - } - - public void setStation_name(String station_name) { - this.station_name = station_name; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java deleted file mode 100644 index 74bb434f08484af6c244dd884570c974ab93e634..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -import java.util.List; - -public class TableMetadata { - - private String dbname; - private String tablename; - private List fields; - private List tags; - - public String getDbname() { - return dbname; - } - - public void setDbname(String dbname) { - this.dbname = dbname; - } - - public String getTablename() { - return tablename; - } - - public void setTablename(String tablename) { - this.tablename = tablename; - } - - public List getFields() { - return fields; - } - - public void setFields(List fields) { - this.fields = fields; - } - - public List getTags() { - return tags; - } - - public void setTags(List tags) { - this.tags = tags; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java deleted file mode 100644 index 755ecc0075506c4ba2c99eafa47593a69a50bd84..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -public class TagMetadata { - private String name; - private String type; - - public TagMetadata(String name, String type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java deleted file mode 100644 index cd7de447eade4a126c86c38c703d6a4de444c557..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.domain; - -import com.fasterxml.jackson.annotation.JsonFormat; - -import java.sql.Timestamp; - -public class Weather { - - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS",timezone = "GMT+8") - private Timestamp ts; - - private int temperature; - - private float humidity; - - public Timestamp getTs() { - return ts; - } - - public void setTs(Timestamp ts) { - this.ts = ts; - } - - public int getTemperature() { - return temperature; - } - - public void setTemperature(int temperature) { - this.temperature = temperature; - } - - public float getHumidity() { - return humidity; - } - - public void setHumidity(float humidity) { - this.humidity = humidity; - } -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java deleted file mode 100644 index 3ea63c1760d4aef9e11a076eaff0c19df3a3a73c..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.service; - -import com.taosdata.jdbc.springbootdemo.dao.DatabaseMapper; -import com.taosdata.jdbc.springbootdemo.dao.RainfallMapper; -import com.taosdata.jdbc.springbootdemo.dao.TableMapper; -import com.taosdata.jdbc.springbootdemo.domain.FieldMetadata; -import com.taosdata.jdbc.springbootdemo.domain.Rainfall; -import com.taosdata.jdbc.springbootdemo.domain.TableMetadata; -import com.taosdata.jdbc.springbootdemo.domain.TagMetadata; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Service -public class RainStationService { - - @Autowired - private DatabaseMapper databaseMapper; - @Autowired - private TableMapper tableMapper; - @Autowired - private RainfallMapper rainfallMapper; - - public boolean init() { - databaseMapper.dropDatabase("rainstation"); - - Map map = new HashMap<>(); - map.put("dbname", "rainstation"); - map.put("keep", "36500"); - map.put("days", "30"); - map.put("blocks", "4"); - databaseMapper.creatDatabaseWithParameters(map); - - databaseMapper.useDatabase("rainstation"); - return true; - } - - public boolean createTable() { - TableMetadata tableMetadata = new TableMetadata(); - tableMetadata.setDbname("rainstation"); - tableMetadata.setTablename("monitoring"); - - List fields = new ArrayList<>(); - fields.add(new FieldMetadata("ts", "timestamp")); - fields.add(new FieldMetadata("name", "NCHAR(10)")); - fields.add(new FieldMetadata("code", " BINARY(8)")); - fields.add(new FieldMetadata("rainfall", "float")); - tableMetadata.setFields(fields); - - List tags = new ArrayList<>(); - tags.add(new TagMetadata("station_code", "BINARY(8)")); - tags.add(new TagMetadata("station_name", "NCHAR(10)")); - tableMetadata.setTags(tags); - - tableMapper.createSTable(tableMetadata); - return true; - } - - - public int insert(Rainfall rainfall) { - Map map = new HashMap<>(); - map.put("dbname", "rainstation"); - map.put("table", "S_53646"); - map.put("stable", "monitoring"); - map.put("values", rainfall); - return rainfallMapper.save(map); - } -} \ No newline at end of file diff --git a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java b/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java deleted file mode 100644 index b950a9a4e42e306320415c7be556ba78c2f25a27..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.taosdata.jdbc.springbootdemo.service; - -import com.taosdata.jdbc.springbootdemo.dao.WeatherMapper; -import com.taosdata.jdbc.springbootdemo.domain.Weather; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Service -public class WeatherService { - - @Autowired - private WeatherMapper weatherMapper; - - public boolean init() { - weatherMapper.createDB(); - weatherMapper.createTable(); - return true; - } - - public List query(Long limit, Long offset) { - return weatherMapper.select(limit, offset); - } - - public int save(int temperature, float humidity) { - Weather weather = new Weather(); - weather.setTemperature(temperature); - weather.setHumidity(humidity); - - return weatherMapper.insert(weather); - } - - public int save(List weatherList) { - return weatherMapper.batchInsert(weatherList); - } - -} diff --git a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties index 683884fcdd852fd1a72c685b40c6a8517ed92ff4..4d7e64d10576388827502a459df9e68da2721dbb 100644 --- a/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/springbootdemo/src/main/resources/application.properties @@ -1,24 +1,20 @@ -# datasource config -spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver -spring.datasource.url=jdbc:TAOS://localhost:6030/log +# datasource config - JDBC-JNI +#spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver +#spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 +#spring.datasource.username=root +#spring.datasource.password=taosdata + +# datasource config - JDBC-RESTful +spring.datasource.driver-class-name=com.taosdata.jdbc.rs.RestfulDriver +spring.datasource.url=jdbc:TAOS-RS://master:6041/test?timezone=UTC-8&charset=UTF-8&locale=en_US.UTF-8 spring.datasource.username=root spring.datasource.password=taosdata spring.datasource.druid.initial-size=5 spring.datasource.druid.min-idle=5 spring.datasource.druid.max-active=5 -# max wait time for get connection, ms -spring.datasource.druid.max-wait=60000 - +spring.datasource.druid.max-wait=30000 spring.datasource.druid.validation-query=select server_status(); -spring.datasource.druid.validation-query-timeout=5000 -spring.datasource.druid.test-on-borrow=false -spring.datasource.druid.test-on-return=false -spring.datasource.druid.test-while-idle=true -spring.datasource.druid.time-between-eviction-runs-millis=60000 -spring.datasource.druid.min-evictable-idle-time-millis=600000 -spring.datasource.druid.max-evictable-idle-time-millis=900000 - #mybatis mybatis.mapper-locations=classpath:mapper/*.xml diff --git a/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java b/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java deleted file mode 100644 index 23a7420dab24a15d9d24341839ba58caa9acb4b9..0000000000000000000000000000000000000000 --- a/tests/examples/JDBC/springbootdemo/src/test/java/com/taosdata/jdbc/springbootdemo/SpringbootdemoApplicationTests.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.taosdata.jdbc.springbootdemo; - -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; - -@SpringBootTest -class SpringbootdemoApplicationTests { - - @Test - void contextLoads() { - } - -} diff --git a/tests/examples/JDBC/taosdemo/pom.xml b/tests/examples/JDBC/taosdemo/pom.xml index 15b22917b6103740ac419e046d5fd8fa79363d25..22c2f3b63e82a3a2cdcc3093c3c43b98ab534a4b 100644 --- a/tests/examples/JDBC/taosdemo/pom.xml +++ b/tests/examples/JDBC/taosdemo/pom.xml @@ -67,9 +67,9 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.15 - system - ${project.basedir}/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar + 2.0.20 + + diff --git a/tests/examples/JDBC/taosdemo/readme.md b/tests/examples/JDBC/taosdemo/readme.md index a4b6e297691c4981fc36a06679d085c183e890a1..451fa2960adb98e2deb8499732aefde11f4810a1 100644 --- a/tests/examples/JDBC/taosdemo/readme.md +++ b/tests/examples/JDBC/taosdemo/readme.md @@ -1,3 +1,13 @@ +``` +cd tests/examples/JDBC/taosdemo +mvn clean package -Dmaven.test.skip=true +# 先建表,再插入的 +java -jar target/taosdemo-2.0-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable true -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 +# 不建表,直接插入的 +java -jar target/taosdemo-2.0-jar-with-dependencies.jar -host [hostname] -database [database] -doCreateTable false -superTableSQL "create table weather(ts timestamp, f1 int) tags(t1 nchar(4))" -numOfTables 1000 -numOfRowsPerTable 100000000 -numOfThreadsForInsert 10 -numOfTablesPerSQL 10 -numOfValuesPerSQL 100 +``` + 需求: 1. 可以读lowa的配置文件 -2. 支持对JNI方式和Restful方式的taos-driver \ No newline at end of file +2. 支持JDBC-JNI和JDBC-restful +3. 读取配置文件,持续执行查询 \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java index b9a22a1ef75962159bd33a8e525898a69b67a911..c361df82b0aebb0d804b1a0982a0c1cf44ef5953 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -4,6 +4,7 @@ import com.taosdata.taosdemo.components.DataSourceFactory; import com.taosdata.taosdemo.components.JdbcTaosdemoConfig; import com.taosdata.taosdemo.domain.SuperTableMeta; import com.taosdata.taosdemo.service.DatabaseService; +import com.taosdata.taosdemo.service.SqlExecuteTask; import com.taosdata.taosdemo.service.SubTableService; import com.taosdata.taosdemo.service.SuperTableService; import com.taosdata.taosdemo.service.data.SuperTableMetaGenerator; @@ -19,22 +20,33 @@ import java.util.Map; public class TaosDemoApplication { - private static Logger logger = Logger.getLogger(TaosDemoApplication.class); + private static final Logger logger = Logger.getLogger(TaosDemoApplication.class); public static void main(String[] args) throws IOException { // 读配置参数 JdbcTaosdemoConfig config = new JdbcTaosdemoConfig(args); boolean isHelp = Arrays.asList(args).contains("--help"); if (isHelp || config.host == null || config.host.isEmpty()) { -// if (isHelp) { JdbcTaosdemoConfig.printHelp(); System.exit(0); } // 初始化 final DataSource dataSource = DataSourceFactory.getInstance(config.host, config.port, config.user, config.password); + if (config.executeSql != null && !config.executeSql.isEmpty() && !config.executeSql.replaceAll("\\s", "").isEmpty()) { + Thread task = new Thread(new SqlExecuteTask(dataSource, config.executeSql)); + task.start(); + try { + task.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return; + } + final DatabaseService databaseService = new DatabaseService(dataSource); final SuperTableService superTableService = new SuperTableService(dataSource); final SubTableService subTableService = new SubTableService(dataSource); + // 创建数据库 long start = System.currentTimeMillis(); Map databaseParam = new HashMap<>(); @@ -75,7 +87,7 @@ public class TaosDemoApplication { } } end = System.currentTimeMillis(); - logger.error(">>> create table time cost : " + (end - start) + " ms."); + logger.info(">>> create table time cost : " + (end - start) + " ms."); /**********************************************************************************/ // 插入 long tableSize = config.numOfTables; @@ -90,7 +102,11 @@ public class TaosDemoApplication { // multi threads to insert int affectedRows = subTableService.insertMultiThreads(superTableMeta, threadSize, tableSize, startTime, gap, config); end = System.currentTimeMillis(); - logger.error("insert " + affectedRows + " rows, time cost: " + (end - start) + " ms"); + logger.info("insert " + affectedRows + " rows, time cost: " + (end - start) + " ms"); + /**********************************************************************************/ + // 查询 + + /**********************************************************************************/ // 删除表 if (config.dropTable) { @@ -108,5 +124,4 @@ public class TaosDemoApplication { return startTime; } - } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java index c96d6f8bed68e9bb67d959ddb1d7531b4cbadeb3..a7d08e96ea373c4773e872bcaf9b3a7b98d5bf9a 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/DataSourceFactory.java @@ -23,7 +23,6 @@ public class DataSourceFactory { properties.load(is); HikariConfig config = new HikariConfig(); - if (properties.containsKey("jdbc.driver")) { // String driverName = properties.getProperty("jdbc.driver"); // System.out.println(">>> load driver : " + driverName); diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java index 971c10dee2889543e95a70b244ea3cda462df3a6..974a2755a5a029d3a5fc681bc8c59b0aca1a7ca4 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/components/JdbcTaosdemoConfig.java @@ -42,7 +42,7 @@ public final class JdbcTaosdemoConfig { public int rate = 10; public long range = 1000l; // select task - + public String executeSql; // drop task public boolean dropTable = false; @@ -89,7 +89,7 @@ public final class JdbcTaosdemoConfig { System.out.println("-rate The proportion of data out of order. effective only if order is 1. min 0, max 100, default is 10"); System.out.println("-range The range of data out of order. effective only if order is 1. default is 1000 ms"); // query task -// System.out.println("-sqlFile The select sql file"); + System.out.println("-executeSql execute a specific sql."); // drop task System.out.println("-dropTable Drop data before quit. Default is false"); System.out.println("--help Give this help list"); @@ -207,6 +207,9 @@ public final class JdbcTaosdemoConfig { range = Integer.parseInt(args[++i]); } // select task + if ("-executeSql".equals(args[i]) && i < args.length - 1) { + executeSql = args[++i]; + } // drop task if ("-dropTable".equals(args[i]) && i < args.length - 1) { diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java index 69bae160f6285bb9d1a94357cc2279ebfca22eca..421a2dea1f4b49786d57b5579ca849976708791e 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java @@ -21,27 +21,27 @@ public class DatabaseMapperImpl implements DatabaseMapper { public void createDatabase(String dbname) { String sql = "create database if not exists " + dbname; jdbcTemplate.execute(sql); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); } @Override public void dropDatabase(String dbname) { String sql = "drop database if exists " + dbname; jdbcTemplate.update(sql); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); } @Override public void createDatabaseWithParameters(Map map) { String sql = SqlSpeller.createDatabase(map); jdbcTemplate.execute(sql); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); } @Override public void useDatabase(String dbname) { String sql = "use " + dbname; jdbcTemplate.execute(sql); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); } } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java index e3a6691430f8b5232660ed6edeab01318db30ef1..90b0990a2bf2e9a9bd2738deec17a284c0868280 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java @@ -21,14 +21,14 @@ public class SubTableMapperImpl implements SubTableMapper { @Override public void createUsingSuperTable(SubTableMeta subTableMeta) { String sql = SqlSpeller.createTableUsingSuperTable(subTableMeta); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); jdbcTemplate.execute(sql); } @Override public int insertOneTableMultiValues(SubTableValue subTableValue) { String sql = SqlSpeller.insertOneTableMultiValues(subTableValue); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); int affectRows = 0; try { @@ -42,7 +42,7 @@ public class SubTableMapperImpl implements SubTableMapper { @Override public int insertOneTableMultiValuesUsingSuperTable(SubTableValue subTableValue) { String sql = SqlSpeller.insertOneTableMultiValuesUsingSuperTable(subTableValue); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); int affectRows = 0; try { @@ -56,7 +56,7 @@ public class SubTableMapperImpl implements SubTableMapper { @Override public int insertMultiTableMultiValues(List tables) { String sql = SqlSpeller.insertMultiSubTableMultiValues(tables); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); int affectRows = 0; try { affectRows = jdbcTemplate.update(sql); @@ -69,7 +69,7 @@ public class SubTableMapperImpl implements SubTableMapper { @Override public int insertMultiTableMultiValuesUsingSuperTable(List tables) { String sql = SqlSpeller.insertMultiTableMultiValuesUsingSuperTable(tables); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); int affectRows = 0; try { affectRows = jdbcTemplate.update(sql); diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java index a293de5100d9af33bc88ad5f02ae632333a5b5b8..efa9a1f39ea41dd1aba65ab610eae095a3164533 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java @@ -18,14 +18,14 @@ public class SuperTableMapperImpl implements SuperTableMapper { @Override public void createSuperTable(SuperTableMeta tableMetadata) { String sql = SqlSpeller.createSuperTable(tableMetadata); - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); jdbcTemplate.execute(sql); } @Override public void dropSuperTable(String database, String name) { String sql = "drop table if exists " + database + "." + name; - logger.info("SQL >>> " + sql); + logger.debug("SQL >>> " + sql); jdbcTemplate.execute(sql); } } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java index 77415619f02a1a0650448e9a8856fcb8d33e4921..b049fbe197034ffcd8801b9c4f5e5ff8dbbcc0e0 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java @@ -1,43 +1,49 @@ package com.taosdata.taosdemo.dao; -import com.taosdata.taosdemo.dao.TableMapper; import com.taosdata.taosdemo.domain.TableMeta; import com.taosdata.taosdemo.domain.TableValue; import com.taosdata.taosdemo.utils.SqlSpeller; +import org.apache.log4j.Logger; import org.springframework.jdbc.core.JdbcTemplate; import java.util.List; public class TableMapperImpl implements TableMapper { + private static final Logger logger = Logger.getLogger(TableMapperImpl.class); private JdbcTemplate template; @Override public void create(TableMeta tableMeta) { String sql = SqlSpeller.createTable(tableMeta); + logger.debug("SQL >>> " + sql); template.execute(sql); } @Override public int insertOneTableMultiValues(TableValue values) { String sql = SqlSpeller.insertOneTableMultiValues(values); + logger.debug("SQL >>> " + sql); return template.update(sql); } @Override public int insertOneTableMultiValuesWithColumns(TableValue values) { String sql = SqlSpeller.insertOneTableMultiValuesWithColumns(values); + logger.debug("SQL >>> " + sql); return template.update(sql); } @Override public int insertMultiTableMultiValues(List tables) { String sql = SqlSpeller.insertMultiTableMultiValues(tables); + logger.debug("SQL >>> " + sql); return template.update(sql); } @Override public int insertMultiTableMultiValuesWithColumns(List tables) { String sql = SqlSpeller.insertMultiTableMultiValuesWithColumns(tables); + logger.debug("SQL >>> " + sql); return template.update(sql); } } diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java new file mode 100644 index 0000000000000000000000000000000000000000..efabff6afe904516ad9682cd7197412dc02765ef --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/QueryService.java @@ -0,0 +1,104 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.jdbc.utils.SqlSyntaxValidator; + +import javax.sql.DataSource; +import java.sql.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +public class QueryService { + + private final DataSource dataSource; + + public QueryService(DataSource dataSource) { + this.dataSource = dataSource; + } + + /* only select or show SQL Statement is valid for executeQuery */ + public Boolean[] areValidQueries(String[] sqls) { + Boolean[] ret = new Boolean[sqls.length]; + for (int i = 0; i < sqls.length; i++) { + ret[i] = true; + if (!SqlSyntaxValidator.isValidForExecuteQuery(sqls[i])) { + ret[i] = false; + continue; + } + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + stmt.executeQuery(sqls[i]); + } catch (SQLException e) { + ret[i] = false; + continue; + } + } + return ret; + } + + public String[] generateSuperTableQueries(String dbName) { + List sqls = new ArrayList<>(); + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + stmt.execute("use " + dbName); + ResultSet rs = stmt.executeQuery("show stables"); + while (rs.next()) { + String name = rs.getString("name"); + sqls.add("select count(*) from " + dbName + "." + name); + sqls.add("select first(*) from " + dbName + "." + name); + sqls.add("select last(*) from " + dbName + "." + name); + sqls.add("select last_row(*) from " + dbName + "." + name); + } + } catch (SQLException e) { + e.printStackTrace(); + } + String[] sqlArr = new String[sqls.size()]; + return sqls.toArray(sqlArr); + } + + public void querySuperTable(String[] sqls, int interval, int threadCount, long queryTimes) { + List threads = IntStream.range(0, threadCount).mapToObj(i -> new Thread(() -> { + // do query + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + long count = queryTimes; + if (count == 0) + count = Long.MAX_VALUE; + while (count > 0) { + for (String sql : sqls) { + long start = System.currentTimeMillis(); + ResultSet rs = stmt.executeQuery(sql); + printResultSet(rs); + long end = System.currentTimeMillis(); + long timecost = end - start; + if (interval - timecost > 0) { + TimeUnit.MILLISECONDS.sleep(interval - timecost); + } + } + count--; + } + + } catch (SQLException | InterruptedException e) { + e.printStackTrace(); + } + + })).collect(Collectors.toList()); + threads.stream().forEach(Thread::start); + for (Thread thread : threads) { + try { + thread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + private void printResultSet(ResultSet rs) throws SQLException { + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + System.out.print(meta.getColumnLabel(i) + ": " + rs.getString(i) + "\t"); + } + System.out.println(); + } + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java new file mode 100644 index 0000000000000000000000000000000000000000..ff2e4d0af068a10e62933837817d2d2df0712a4c --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SqlExecuteTask.java @@ -0,0 +1,36 @@ +package com.taosdata.taosdemo.service; + +import com.taosdata.taosdemo.utils.Printer; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +public class SqlExecuteTask implements Runnable { + private final DataSource dataSource; + private final String sql; + + public SqlExecuteTask(DataSource dataSource, String sql) { + this.dataSource = dataSource; + this.sql = sql; + } + + @Override + public void run() { + try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement()) { + long start = System.currentTimeMillis(); + boolean execute = stmt.execute(sql); + long end = System.currentTimeMillis(); + if (execute) { + ResultSet rs = stmt.getResultSet(); + Printer.printResult(rs); + } else { + Printer.printSql(sql, true, (end - start)); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java index b4ad2d17e58a3f7c04665707f0cd3e7327d7c16c..2504fdb0b4cd48ec263d94ec377e1bb8902ea9a7 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/TableService.java @@ -2,7 +2,6 @@ package com.taosdata.taosdemo.service; import com.taosdata.taosdemo.dao.TableMapper; import com.taosdata.taosdemo.domain.TableMeta; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.List; diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java new file mode 100644 index 0000000000000000000000000000000000000000..a4627463ec5cd1bbccdb64b67506ba38f712de8f --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/utils/Printer.java @@ -0,0 +1,27 @@ +package com.taosdata.taosdemo.utils; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; + +public class Printer { + + public static void printResult(ResultSet resultSet) throws SQLException { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String columnLabel = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + System.out.printf("%s: %s\t", columnLabel, value); + } + System.out.println(); + } + } + + public static void printSql(String sql, boolean succeed, long cost) { + System.out.println("[ " + (succeed ? "OK" : "ERROR!") + " ] time cost: " + cost + " ms, execute statement ====> " + sql); + } + + private Printer() { + } +} diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/application.properties b/tests/examples/JDBC/taosdemo/src/main/resources/application.properties index 6fd38f1762e6879422cf26fe2dc617caeb5e297c..488185196f1d2325fd9896d30068cbb202180a3f 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/application.properties +++ b/tests/examples/JDBC/taosdemo/src/main/resources/application.properties @@ -1,5 +1,5 @@ jdbc.driver=com.taosdata.jdbc.rs.RestfulDriver #jdbc.driver=com.taosdata.jdbc.TSDBDriver -hikari.maximum-pool-size=1 -hikari.minimum-idle=1 +hikari.maximum-pool-size=20 +hikari.minimum-idle=20 hikari.max-lifetime=0 \ No newline at end of file diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/insert.json b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json index a7bd87e6d3bbf9f6ec1b0a68d31c4da6c620c994..35c77731758c7c8de92f050803a537c776578f3d 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/insert.json +++ b/tests/examples/JDBC/taosdemo/src/main/resources/insert.json @@ -38,7 +38,7 @@ "insert_rows": 100, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, - "rows_per_tbl": 3, + "interlace_rows": 3, "max_sql_len": 1024, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar b/tests/examples/JDBC/taosdemo/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar deleted file mode 100644 index 58508b92e5afec7e400691d213940536e8abe5f6..0000000000000000000000000000000000000000 Binary files a/tests/examples/JDBC/taosdemo/src/main/resources/lib/taos-jdbcdriver-2.0.15-dist.jar and /dev/null differ diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties b/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties index b2a9586ea78e2d55a0091097097988b038267405..352545854d0f68f07e4da6a98504ffa7754b03b1 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties +++ b/tests/examples/JDBC/taosdemo/src/main/resources/log4j.properties @@ -1,5 +1,5 @@ ### 设置### -log4j.rootLogger=error,stdout +log4j.rootLogger=info,stdout ### 输出信息到控制抬 ### log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target=System.out diff --git a/tests/examples/JDBC/taosdemo/src/main/resources/query.json b/tests/examples/JDBC/taosdemo/src/main/resources/query.json index 53d0b319212196257aa3e84be1221bd6e2bd0d8d..cc6900d77c3941e6af3274efdfe782c42a557990 100644 --- a/tests/examples/JDBC/taosdemo/src/main/resources/query.json +++ b/tests/examples/JDBC/taosdemo/src/main/resources/query.json @@ -7,10 +7,10 @@ "password": "taosdata", "databases": "db01", "super_table_query": - {"rate":1, "concurrent":1, + {"rate":1, "concurrent":1,"time":10000, "sqls": [{"sql": "select count(*) from stb01", "result": "./query_res0.txt"}] }, - "sub_table_query": + "sub_table_query": {"stblname": "stb01", "rate":1, "threads":1, "sqls": [{"sql": "select count(*) from xxxx", "result": "./query_res1.txt"}] } diff --git a/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java new file mode 100644 index 0000000000000000000000000000000000000000..f2ad25710c1a82136d6316ed69e379bc3925897d --- /dev/null +++ b/tests/examples/JDBC/taosdemo/src/test/java/com/taosdata/taosdemo/service/QueryServiceTest.java @@ -0,0 +1,41 @@ +package com.taosdata.taosdemo.service; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.junit.BeforeClass; +import org.junit.Test; + +public class QueryServiceTest { + private static QueryService queryService; + + @Test + public void areValidQueries() { + + } + + @Test + public void generateSuperTableQueries() { + String[] sqls = queryService.generateSuperTableQueries("restful_test"); + for (String sql : sqls) { + System.out.println(sql); + } + } + + @Test + public void querySuperTable() { + String[] sqls = queryService.generateSuperTableQueries("restful_test"); + queryService.querySuperTable(sqls, 1000, 10, 10); + } + + @BeforeClass + public static void beforeClass() throws ClassNotFoundException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/?charset=UTF-8&locale=en_US.UTF-8&timezone=UTC-8"); + config.setUsername("root"); + config.setPassword("taosdata"); + HikariDataSource dataSource = new HikariDataSource(config); + queryService = new QueryService(dataSource); + } + +} \ No newline at end of file diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 59bcb6eaff349e8b20fa13995d1a7297f9185a50..954fe468b1b8fc88ce93fa2474e2f69a33415e6f 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -1,8 +1,18 @@ PROJECT(TDengine) IF (TD_LINUX) + INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(demo apitest.c) + TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) + ADD_EXECUTABLE(epoll epoll.c) + TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) +ENDIF () +IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(demo demo.c) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) + ADD_EXECUTABLE(epoll epoll.c) + TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread ) ENDIF () diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index e08d667d6be42bef3dc4e058fab10ac68f0809e0..930a6075cae2767c0d50ca2e1574d6441430d1b3 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -467,7 +467,6 @@ int main(int argc, char *argv[]) { const char* passwd = "taosdata"; taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - taos_init(); TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { diff --git a/tests/examples/c/asyncdemo.c b/tests/examples/c/asyncdemo.c index c6cc89b31d6280c45ea30b33509eed5ebdf0dc08..d711ce22c1e5a22b643bbbcb33468cd91713c861 100644 --- a/tests/examples/c/asyncdemo.c +++ b/tests/examples/c/asyncdemo.c @@ -99,8 +99,6 @@ int main(int argc, char *argv[]) tableList = (STable *)malloc(size); memset(tableList, 0, size); - taos_init(); - taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) taos_error(taos); @@ -165,6 +163,16 @@ int main(int argc, char *argv[]) getchar(); + while(1) { + if (tablesProcessed < numOfTables) { + printf("wait for process finished\n"); + sleep(1); + continue; + } + + break; + } + taos_close(taos); free(tableList); @@ -261,9 +269,6 @@ void taos_select_call_back(void *param, TAOS_RES *tres, int code) if (code == 0 && tres) { // asynchronous API to fetch a batch of records taos_fetch_rows_a(tres, taos_retrieve_call_back, pTable); - - // taos_fetch_row_a is a less efficient way to retrieve records since it call back app for every row - // taos_fetch_row_a(tres, taos_fetch_row_call_back, pTable); } else { printf("%s select failed, code:%d\n", pTable->name, code); diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 54e81d33b9c7a948dc75996828bc07634d4664ae..e074e6496607bf2046496c7b6766b6f3dedf9dbc 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -61,8 +61,6 @@ int main(int argc, char *argv[]) { return 0; } - // init TAOS - taos_init(); TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) { printf("failed to connect to server, reason:%s\n", "null taos"/*taos_errstr(taos)*/); @@ -86,23 +84,22 @@ void Test(TAOS *taos, char *qstr, int index) { int i = 0; for (i = 0; i < 10; ++i) { - sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); + sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", (uint64_t)(1546300800000 + i * 1000), i, i, i, i*10000000, i*1.0, i*2.0, "hello"); printf("qstr: %s\n", qstr); // note: how do you wanna do if taos_query returns non-NULL // if (taos_query(taos, qstr)) { // printf("insert row: %i, reason:%s\n", i, taos_errstr(taos)); // } - TAOS_RES *result = taos_query(taos, qstr); - if (result) { - printf("insert row: %i\n", i); - } else { - printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/); - taos_free_result(result); + TAOS_RES *result1 = taos_query(taos, qstr); + if (result1 == NULL || taos_errno(result1) != 0) { + printf("failed to insert row, reason:%s\n", taos_errstr(result1)); + taos_free_result(result1); exit(1); + } else { + printf("insert row: %i\n", i); } - taos_free_result(result); - + taos_free_result(result1); } printf("success to insert rows, total %d rows\n", i); diff --git a/tests/examples/c/epoll.c b/tests/examples/c/epoll.c new file mode 100644 index 0000000000000000000000000000000000000000..de7c5989d1ecdaec578e4322ac49491587ad851b --- /dev/null +++ b/tests/examples/c/epoll.c @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// how to use to do a pressure-test upon eok +// tester: cat /dev/urandom | nc -c +// testee: ./debug/build/bin/epoll -l > /dev/null +// compare against: nc -l > /dev/null +// monitor and compare : glances + +#ifdef __APPLE__ +#include "eok.h" +#else // __APPLE__ +#include +#endif // __APPLE__ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) +#define A(statement, fmt, ...) do { \ + if (statement) break; \ + fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + #statement, errno, strerror(errno), \ + ##__VA_ARGS__); \ + abort(); \ +} while (0) + +#define E(fmt, ...) do { \ + fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ + basename(__FILE__), __LINE__, __func__, \ + errno, strerror(errno), \ + ##__VA_ARGS__); \ +} while (0) + +#include "os.h" + +typedef struct ep_s ep_t; +struct ep_s { + int ep; + + pthread_mutex_t lock; + int sv[2]; // 0 for read, 1 for write; + pthread_t thread; + + volatile unsigned int stopping:1; + volatile unsigned int waiting:1; + volatile unsigned int wakenup:1; +}; + +static int ep_dummy = 0; + +static ep_t* ep_create(void); +static void ep_destroy(ep_t *ep); +static void* routine(void* arg); +static int open_listen(unsigned short port); + +typedef struct fde_s fde_t; +struct fde_s { + int skt; + void (*on_event)(ep_t *ep, struct epoll_event *events, fde_t *client); +}; + +static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client); +static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client); + +#define usage(arg0, fmt, ...) do { \ + if (fmt[0]) { \ + fprintf(stderr, "" fmt "\n", ##__VA_ARGS__); \ + } \ + fprintf(stderr, "usage:\n"); \ + fprintf(stderr, " %s -l : specify listenning port\n", arg0); \ +} while (0) + +int main(int argc, char *argv[]) { + char *prg = basename(argv[0]); + if (argc==1) { + usage(prg, ""); + return 0; + } + ep_t* ep = ep_create(); + A(ep, "failed"); + for (int i=1; i=argc) { + usage(prg, "expecting after -l, but got nothing"); + return 1; // confirmed potential leakage + } + arg = argv[i]; + int port = atoi(arg); + int skt = open_listen(port); + if (skt==-1) continue; + fde_t *client = (fde_t*)calloc(1, sizeof(*client)); + if (!client) { + E("out of memory"); + close(skt); + continue; + } + client->skt = skt; + client->on_event = listen_event; + struct epoll_event ev = {0}; + ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ev.data.ptr = client; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); + continue; + } + usage(prg, "unknown argument: [%s]", arg); + return 1; + } + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + while ((linelen = getline(&line, &linecap, stdin)) > 0) { + line[strlen(line)-1] = '\0'; + if (0==strcmp(line, "exit")) break; + if (0==strcmp(line, "quit")) break; + if (line==strstr(line, "close")) { + int fd = 0; + sscanf(line, "close %d", &fd); + if (fd<=2) { + fprintf(stderr, "fd [%d] invalid\n", fd); + continue; + } + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, fd, NULL), ""); + continue; + } + if (strlen(line)==0) continue; + fprintf(stderr, "unknown cmd:[%s]\n", line); + } + ep_destroy(ep); + D(""); + return 0; +} + +ep_t* ep_create(void) { + ep_t *ep = (ep_t*)calloc(1, sizeof(*ep)); + A(ep, "out of memory"); + A(-1!=(ep->ep = epoll_create(1)), ""); + ep->sv[0] = -1; + ep->sv[1] = -1; + A(0==socketpair(AF_LOCAL, SOCK_STREAM, 0, ep->sv), ""); + A(0==pthread_mutex_init(&ep->lock, NULL), ""); + A(0==pthread_mutex_lock(&ep->lock), ""); + struct epoll_event ev = {0}; + ev.events = EPOLLIN; + ev.data.ptr = &ep_dummy; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, ep->sv[0], &ev), ""); + A(0==pthread_create(&ep->thread, NULL, routine, ep), ""); + A(0==pthread_mutex_unlock(&ep->lock), ""); + return ep; +} + +static void ep_destroy(ep_t *ep) { + A(ep, "invalid argument"); + ep->stopping = 1; + A(1==send(ep->sv[1], "1", 1, 0), ""); + A(0==pthread_join(ep->thread, NULL), ""); + A(0==pthread_mutex_destroy(&ep->lock), ""); + A(0==close(ep->sv[0]), ""); + A(0==close(ep->sv[1]), ""); + A(0==close(ep->ep), ""); + free(ep); +} + +static void* routine(void* arg) { + A(arg, "invalid argument"); + ep_t *ep = (ep_t*)arg; + + while (!ep->stopping) { + struct epoll_event evs[10]; + memset(evs, 0, sizeof(evs)); + + A(0==pthread_mutex_lock(&ep->lock), ""); + A(ep->waiting==0, "internal logic error"); + ep->waiting = 1; + A(0==pthread_mutex_unlock(&ep->lock), ""); + + int r = epoll_wait(ep->ep, evs, sizeof(evs)/sizeof(evs[0]), -1); + A(r>0, "indefinite epoll_wait shall not timeout:[%d]", r); + + A(0==pthread_mutex_lock(&ep->lock), ""); + A(ep->waiting==1, "internal logic error"); + ep->waiting = 0; + A(0==pthread_mutex_unlock(&ep->lock), ""); + + for (int i=0; idata.ptr == &ep_dummy) { + char c = '\0'; + A(1==recv(ep->sv[0], &c, 1, 0), "internal logic error"); + A(0==pthread_mutex_lock(&ep->lock), ""); + ep->wakenup = 0; + A(0==pthread_mutex_unlock(&ep->lock), ""); + continue; + } + A(ev->data.ptr, "internal logic error"); + fde_t *client = (fde_t*)ev->data.ptr; + client->on_event(ep, ev, client); + continue; + } + } + return NULL; +} + +static int open_listen(unsigned short port) { + int r = 0; + int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (skt==-1) { + E("socket() failed"); + return -1; + } + do { + struct sockaddr_in si = {0}; + si.sin_family = AF_INET; + si.sin_addr.s_addr = inet_addr("0.0.0.0"); + si.sin_port = htons(port); + r = bind(skt, (struct sockaddr*)&si, sizeof(si)); + if (r) { + E("bind(%u) failed", port); + break; + } + r = listen(skt, 100); + if (r) { + E("listen() failed"); + break; + } + memset(&si, 0, sizeof(si)); + socklen_t len = sizeof(si); + r = getsockname(skt, (struct sockaddr *)&si, &len); + if (r) { + E("getsockname() failed"); + } + A(len==sizeof(si), "internal logic error"); + D("listenning at: %d", ntohs(si.sin_port)); + return skt; + } while (0); + close(skt); + return -1; +} + +static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client) { + A(ev->events & EPOLLIN, "internal logic error"); + struct sockaddr_in si = {0}; + socklen_t silen = sizeof(si); + int skt = accept(client->skt, (struct sockaddr*)&si, &silen); + A(skt!=-1, "internal logic error"); + fde_t *server = (fde_t*)calloc(1, sizeof(*server)); + if (!server) { + close(skt); + return; + } + server->skt = skt; + server->on_event = null_event; + struct epoll_event ee = {0}; + ee.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; + ee.data.ptr = server; + A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ee), ""); +} + +static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client) { + if (ev->events & EPOLLIN) { + char buf[8192]; + int n = recv(client->skt, buf, sizeof(buf), 0); + A(n>=0 && n<=sizeof(buf), "internal logic error:[%d]", n); + A(n==fwrite(buf, 1, n, stdout), "internal logic error"); + } + if (ev->events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { + A(0==pthread_mutex_lock(&ep->lock), ""); + A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, client->skt, NULL), ""); + A(0==pthread_mutex_unlock(&ep->lock), ""); + close(client->skt); + client->skt = -1; + client->on_event = NULL; + free(client); + } +} + diff --git a/tests/examples/c/makefile b/tests/examples/c/makefile index 7293a22c11d7c6049c220ca0951b5969fa344114..b06fe551dbde4b16e3ea197ca4c2bc1711ef63bb 100644 --- a/tests/examples/c/makefile +++ b/tests/examples/c/makefile @@ -6,7 +6,7 @@ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ - -msse4.2 -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 + -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 all: $(TARGET) diff --git a/tests/examples/c/prepare.c b/tests/examples/c/prepare.c index 7a70b744ee8561459318aa456160a54b8c6270a8..13d71beea6156aa32677c20ccc2d222d28826b24 100644 --- a/tests/examples/c/prepare.c +++ b/tests/examples/c/prepare.c @@ -22,9 +22,6 @@ int main(int argc, char *argv[]) return 0; } - // init TAOS - taos_init(); - taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) { printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); diff --git a/tests/examples/c/stream.c b/tests/examples/c/stream.c index 060f5b84ff276579019d3278552e424b2a4198e9..30a790f061cd8ef2b870a371c2cadfb0e2a413c1 100644 --- a/tests/examples/c/stream.c +++ b/tests/examples/c/stream.c @@ -54,9 +54,6 @@ int main(int argc, char *argv[]) exit(0); } - // init TAOS - taos_init(); - strcpy(db_name, argv[2]); strcpy(tbl_name, argv[3]); diff --git a/tests/examples/c/subscribe.c b/tests/examples/c/subscribe.c index cdd8ddaf7f6c4d2e5088ef36cc00ad77a0c8ebc9..1d3533fa5ee89f7425a9b0563e738b4e0703e6c8 100644 --- a/tests/examples/c/subscribe.c +++ b/tests/examples/c/subscribe.c @@ -216,9 +216,6 @@ int main(int argc, char *argv[]) { } } - // init TAOS - taos_init(); - TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); diff --git a/tests/examples/go/taosdemo.go b/tests/examples/go/taosdemo.go index 2c3a7d09b68d84feea1ae2771b90643dbbfbc063..003f5aeddccd7f988b62227815a98cde8610f311 100644 --- a/tests/examples/go/taosdemo.go +++ b/tests/examples/go/taosdemo.go @@ -16,45 +16,47 @@ package main import ( "database/sql" + "flag" "fmt" - _ "github.com/taosdata/driver-go/taosSql" + "math/rand" "os" - "sync" "runtime" "strconv" + "sync" "time" - "flag" - "math/rand" + + _ "github.com/taosdata/driver-go/taosSql" + //"golang.org/x/sys/unix" ) const ( - maxLocationSize = 32 - maxSqlBufSize = 65480 + maxLocationSize = 32 + //maxSqlBufSize = 65480 ) -var locations = [maxLocationSize]string { - "Beijing", "Shanghai", "Guangzhou", "Shenzhen", - "HangZhou", "Tianjin", "Wuhan", "Changsha", - "Nanjing", "Xian"} +var locations = [maxLocationSize]string{ + "Beijing", "Shanghai", "Guangzhou", "Shenzhen", + "HangZhou", "Tianjin", "Wuhan", "Changsha", + "Nanjing", "Xian"} type config struct { - hostName string - serverPort int - user string - password string - dbName string - supTblName string - tablePrefix string - numOftables int - numOfRecordsPerTable int - numOfRecordsPerReq int - numOfThreads int - startTimestamp string - startTs int64 - - keep int - days int + hostName string + serverPort int + user string + password string + dbName string + supTblName string + tablePrefix string + numOftables int + numOfRecordsPerTable int + numOfRecordsPerReq int + numOfThreads int + startTimestamp string + startTs int64 + + keep int + days int } var configPara config @@ -62,7 +64,7 @@ var taosDriverName = "taosSql" var url string func init() { - flag.StringVar(&configPara.hostName, "h", "127.0.0.1","The host to connect to TDengine server.") + flag.StringVar(&configPara.hostName, "h", "127.0.0.1", "The host to connect to TDengine server.") flag.IntVar(&configPara.serverPort, "p", 6030, "The TCP/IP port number to use for the connection to TDengine server.") flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.") flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.") @@ -80,14 +82,14 @@ func init() { configPara.supTblName = "meters" startTs, err := time.ParseInLocation("2006-01-02 15:04:05", configPara.startTimestamp, time.Local) - if err==nil { - configPara.startTs = startTs.UnixNano() / 1e6 + if err == nil { + configPara.startTs = startTs.UnixNano() / 1e6 } } func printAllArgs() { fmt.Printf("\n============= args parse result: =============\n") - fmt.Printf("hostName: %v\n", configPara.hostName) + fmt.Printf("hostName: %v\n", configPara.hostName) fmt.Printf("serverPort: %v\n", configPara.serverPort) fmt.Printf("usr: %v\n", configPara.user) fmt.Printf("password: %v\n", configPara.password) @@ -104,10 +106,10 @@ func printAllArgs() { func main() { printAllArgs() fmt.Printf("Please press enter key to continue....\n") - fmt.Scanln() + _, _ = fmt.Scanln() url = "root:taosdata@/tcp(" + configPara.hostName + ":" + strconv.Itoa(configPara.serverPort) + ")/" - //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) + //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) // open connect to taos server //db, err := sql.Open(taosDriverName, url) //if err != nil { @@ -115,7 +117,7 @@ func main() { // os.Exit(1) //} //defer db.Close() - rand.Seed(time.Now().Unix()) + rand.Seed(time.Now().Unix()) createDatabase(configPara.dbName, configPara.supTblName) fmt.Printf("======== create database success! ========\n\n") @@ -138,7 +140,7 @@ func main() { func createDatabase(dbName string, supTblName string) { db, err := sql.Open(taosDriverName, url) if err != nil { - fmt.Println("Open database error: %s\n", err) + fmt.Printf("Open database error: %s\n", err) os.Exit(1) } defer db.Close() @@ -165,27 +167,27 @@ func createDatabase(dbName string, supTblName string) { checkErr(err, sqlStr) } -func multiThreadCreateTable(threads int, ntables int, dbName string, tablePrefix string) { +func multiThreadCreateTable(threads int, nTables int, dbName string, tablePrefix string) { st := time.Now().UnixNano() - if (threads < 1) { - threads = 1; + if threads < 1 { + threads = 1 } - a := ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; + a := nTables / threads + if a < 1 { + threads = nTables + a = 1 } - b := ntables % threads; + b := nTables % threads - last := 0; + last := 0 endTblId := 0 wg := sync.WaitGroup{} for i := 0; i < threads; i++ { startTblId := last - if (i < b ) { + if i < b { endTblId = last + a } else { endTblId = last + a - 1 @@ -206,42 +208,43 @@ func createTable(dbName string, childTblPrefix string, startTblId int, endTblId db, err := sql.Open(taosDriverName, url) if err != nil { - fmt.Println("Open database error: %s\n", err) + fmt.Printf("Open database error: %s\n", err) os.Exit(1) } defer db.Close() - for i := startTblId; i <= endTblId; i++ { - sqlStr := "create table if not exists " + dbName + "." + childTblPrefix + strconv.Itoa(i) + " using " + dbName + ".meters tags('" + locations[i%maxLocationSize] + "', " + strconv.Itoa(i) + ");" - //fmt.Printf("sqlStr: %v\n", sqlStr) - _, err = db.Exec(sqlStr) - checkErr(err, sqlStr) - } - wg.Done() - runtime.Goexit() + for i := startTblId; i <= endTblId; i++ { + sqlStr := "create table if not exists " + dbName + "." + childTblPrefix + strconv.Itoa(i) + " using " + dbName + ".meters tags('" + locations[i%maxLocationSize] + "', " + strconv.Itoa(i) + ");" + //fmt.Printf("sqlStr: %v\n", sqlStr) + _, err = db.Exec(sqlStr) + checkErr(err, sqlStr) + } + wg.Done() + runtime.Goexit() } func generateRowData(ts int64) string { - voltage := rand.Int() % 1000 - current := 200 + rand.Float32() - phase := rand.Float32() - values := "( " + strconv.FormatInt(ts, 10) + ", " + strconv.FormatFloat(float64(current), 'f', 6, 64) + ", " + strconv.Itoa(voltage) + ", " + strconv.FormatFloat(float64(phase), 'f', 6, 64) + " ) " - return values + voltage := rand.Int() % 1000 + current := 200 + rand.Float32() + phase := rand.Float32() + values := "( " + strconv.FormatInt(ts, 10) + ", " + strconv.FormatFloat(float64(current), 'f', 6, 64) + ", " + strconv.Itoa(voltage) + ", " + strconv.FormatFloat(float64(phase), 'f', 6, 64) + " ) " + return values } + func insertData(dbName string, childTblPrefix string, startTblId int, endTblId int, wg *sync.WaitGroup) { //fmt.Printf("subThread[%d]: insert data to table from %d to %d \n", unix.Gettid(), startTblId, endTblId) // windows.GetCurrentThreadId() db, err := sql.Open(taosDriverName, url) if err != nil { - fmt.Println("Open database error: %s\n", err) + fmt.Printf("Open database error: %s\n", err) os.Exit(1) } defer db.Close() - tmpTs := configPara.startTs; + tmpTs := configPara.startTs //rand.New(rand.NewSource(time.Now().UnixNano())) - for tID := startTblId; tID <= endTblId; tID++{ + for tID := startTblId; tID <= endTblId; tID++ { totalNum := 0 for { sqlStr := "insert into " + dbName + "." + childTblPrefix + strconv.Itoa(tID) + " values " @@ -249,13 +252,13 @@ func insertData(dbName string, childTblPrefix string, startTblId int, endTblId i for { tmpTs += 1000 valuesOfRow := generateRowData(tmpTs) - currRowNum += 1 - totalNum += 1 + currRowNum += 1 + totalNum += 1 sqlStr = fmt.Sprintf("%s %s", sqlStr, valuesOfRow) - if (currRowNum >= configPara.numOfRecordsPerReq || totalNum >= configPara.numOfRecordsPerTable) { - break + if currRowNum >= configPara.numOfRecordsPerReq || totalNum >= configPara.numOfRecordsPerTable { + break } } @@ -265,12 +268,12 @@ func insertData(dbName string, childTblPrefix string, startTblId int, endTblId i count, err := res.RowsAffected() checkErr(err, "rows affected") - if (count != int64(currRowNum)) { - fmt.Printf("insert data, expect affected:%d, actual:%d\n", currRowNum, count) + if count != int64(currRowNum) { + fmt.Printf("insert data, expect affected:%d, actual:%d\n", currRowNum, count) os.Exit(1) } - if (totalNum >= configPara.numOfRecordsPerTable) { + if totalNum >= configPara.numOfRecordsPerTable { break } } @@ -279,44 +282,46 @@ func insertData(dbName string, childTblPrefix string, startTblId int, endTblId i wg.Done() runtime.Goexit() } -func multiThreadInsertData(threads int, ntables int, dbName string, tablePrefix string) { + +func multiThreadInsertData(threads int, nTables int, dbName string, tablePrefix string) { st := time.Now().UnixNano() - if (threads < 1) { - threads = 1; + if threads < 1 { + threads = 1 } - a := ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; + a := nTables / threads + if a < 1 { + threads = nTables + a = 1 } - b := ntables % threads; + b := nTables % threads - last := 0; + last := 0 endTblId := 0 wg := sync.WaitGroup{} for i := 0; i < threads; i++ { startTblId := last - if (i < b ) { + if i < b { endTblId = last + a } else { endTblId = last + a - 1 } last = endTblId + 1 wg.Add(1) - go insertData(dbName, tablePrefix, startTblId , endTblId, &wg) + go insertData(dbName, tablePrefix, startTblId, endTblId, &wg) } wg.Wait() et := time.Now().UnixNano() fmt.Printf("insert data spent duration: %6.6fs\n", (float32(et-st))/1e9) } -func selectTest(dbName string, tbPrefix string, supTblName string){ + +func selectTest(dbName string, tbPrefix string, supTblName string) { db, err := sql.Open(taosDriverName, url) if err != nil { - fmt.Println("Open database error: %s\n", err) + fmt.Printf("Open database error: %s\n", err) os.Exit(1) } defer db.Close() @@ -332,12 +337,12 @@ func selectTest(dbName string, tbPrefix string, supTblName string){ fmt.Printf("query sql: %s\n", sqlStr) for rows.Next() { var ( - ts string - current float32 - voltage int - phase float32 - location string - groupid int + ts string + current float32 + voltage int + phase float32 + location string + groupid int ) err := rows.Scan(&ts, ¤t, &voltage, &phase, &location, &groupid) if err != nil { @@ -352,7 +357,7 @@ func selectTest(dbName string, tbPrefix string, supTblName string){ } // select sql 2 - sqlStr = "select avg(voltage), min(voltage), max(voltage) from " + dbName + "." + tbPrefix + strconv.Itoa( rand.Int() % configPara.numOftables) + sqlStr = "select avg(voltage), min(voltage), max(voltage) from " + dbName + "." + tbPrefix + strconv.Itoa(rand.Int()%configPara.numOftables) rows, err = db.Query(sqlStr) checkErr(err, sqlStr) @@ -360,9 +365,9 @@ func selectTest(dbName string, tbPrefix string, supTblName string){ fmt.Printf("\nquery sql: %s\n", sqlStr) for rows.Next() { var ( - voltageAvg float32 - voltageMin int - voltageMax int + voltageAvg float32 + voltageMin int + voltageMax int ) err := rows.Scan(&voltageAvg, &voltageMin, &voltageMax) if err != nil { @@ -385,10 +390,10 @@ func selectTest(dbName string, tbPrefix string, supTblName string){ fmt.Printf("\nquery sql: %s\n", sqlStr) for rows.Next() { var ( - lastTs string - lastCurrent float32 - lastVoltage int - lastPhase float32 + lastTs string + lastCurrent float32 + lastVoltage int + lastPhase float32 ) err := rows.Scan(&lastTs, &lastCurrent, &lastVoltage, &lastPhase) if err != nil { diff --git a/tests/examples/nodejs/nodejsChecker.js b/tests/examples/nodejs/nodejsChecker.js index f838d5cc8465dba70b5372a5d7720a8cff69544a..e634a54ea1c56310bae7d38188590a0d7862f953 100644 --- a/tests/examples/nodejs/nodejsChecker.js +++ b/tests/examples/nodejs/nodejsChecker.js @@ -1,4 +1,5 @@ const taos = require('td2.0-connector'); +//const taos = require('../../../src/connector/nodejs/'); var host = null; diff --git a/tests/examples/python/taosdemo/README.md b/tests/examples/python/taosdemo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d48fffe8ff44fb68a1a147a2c97ca057fb360092 --- /dev/null +++ b/tests/examples/python/taosdemo/README.md @@ -0,0 +1,38 @@ +install build environment +=== +/usr/bin/python3 -m pip install -r requirements.txt + +run python version taosdemo +=== +Usage: ./taosdemo.py [OPTION...] + +Author: Shuduo Sang + + -H, --help Show usage. + + -N, --native flag, Use native interface if set. Default is using RESTful interface. + -h, --host host, The host to connect to TDengine. Default is localhost. + -p, --port port, The TCP/IP port number to use for the connection. Default is 0. + -u, --user user, The user name to use when connecting to the server. Default is 'root'. + -P, --password password, The password to use when connecting to the server. Default is 'taosdata'. + -l, --colsPerRec num_of_columns_per_record, The number of columns per record. Default is 3. + -d, --dbname database, Destination database. Default is 'test'. + -a, --replica replica, Set the replica parameters of the database, Default 1, min: 1, max: 5. + -m, --tbname table_prefix, Table prefix name. Default is 't'. + -M, --stable flag, Use super table. Default is no + -s, --stbname stable_prefix, STable prefix name. Default is 'st' + -Q, --query query, Execute query command. set 'DEFAULT' means select * from each table + -T, --threads num_of_threads, The number of threads. Default is 1. + -C, --processes num_of_processes, The number of threads. Default is 1. + -r, --batch num_of_records_per_req, The number of records per request. Default is 1000. + -t, --numOfTb num_of_tables, The number of tables. Default is 1. + -n, --numOfRec num_of_records_per_table, The number of records per table. Default is 1. + -c, --config config_directory, Configuration directory. Default is '/etc/taos/'. + -x, --inserOnly flag, Insert only flag. + -O, --outOfOrder out of order data insert, 0: In order, 1: Out of order. Default is in order. + -R, --rateOOOO rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50. + -D, --deleteMethod Delete data methods 0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database. + -v, --verbose Print verbose output + -g, --debug Print debug output + -y, --skipPrompt Skip read key for continous test, default is not skip + diff --git a/tests/examples/python/taosdemo/requirements.txt b/tests/examples/python/taosdemo/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..977e8e3726a446e85635764fe9243a3c5416ea0f --- /dev/null +++ b/tests/examples/python/taosdemo/requirements.txt @@ -0,0 +1,28 @@ +## +######## example-requirements.txt ####### +## +####### Requirements without Version Specifiers ###### +requests +multipledispatch +#beautifulsoup4 +## +####### Requirements with Version Specifiers ###### +## See https://www.python.org/dev/peps/pep-0440/#version-specifiers +#docopt == 0.6.1 # Version Matching. Must be version 0.6.1 +#keyring >= 4.1.1 # Minimum version 4.1.1 +#coverage != 3.5 # Version Exclusion. Anything except version 3.5 +#Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.* +## +####### Refer to other requirements files ###### +#-r other-requirements.txt +## +## +####### A particular file ###### +#./downloads/numpy-1.9.2-cp34-none-win32.whl +#http://wxpython.org/Phoenix/snapshot-builds/wxPython_Phoenix-3.0.3.dev1820+49a8884-cp34-none-win_amd64.whl +## +####### Additional Requirements without Version Specifiers ###### +## Same as 1st section, just here to show that you can put things in any order. +#rejected +#green +## diff --git a/tests/examples/python/taosdemo/taosdemo.py b/tests/examples/python/taosdemo/taosdemo.py new file mode 100755 index 0000000000000000000000000000000000000000..d55023bdbf119544a788aa6246c9d63dbf024872 --- /dev/null +++ b/tests/examples/python/taosdemo/taosdemo.py @@ -0,0 +1,797 @@ +#!/usr/bin/python3 +# * Copyright (c) 2019 TAOS Data, Inc. +# * +# * This program is free software: you can use, redistribute, and/or modify +# * it under the terms of the GNU Affero General Public License, version 3 +# * or later ("AGPL"), as published by the Free Software Foundation. +# * +# * This program is distributed in the hope that it will be useful, but WITHOUT +# * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# * FITNESS FOR A PARTICULAR PURPOSE. +# * +# * You should have received a copy of the GNU Affero General Public License +# * along with this program. If not, see . + +# -*- coding: utf-8 -*- + +import sys +import getopt +import requests +import json +import random +import time +import datetime +from multiprocessing import Manager, Pool, Lock +from multipledispatch import dispatch +from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED + + +@dispatch(str, str) +def v_print(msg: str, arg: str): + if verbose: + print(msg % arg) + + +@dispatch(str, str, str) +def v_print(msg: str, arg1: str, arg2: str): + if verbose: + print(msg % (arg1, arg2)) + + +@dispatch(str, str, str, str) +def v_print(msg: str, arg1: str, arg2: str, arg3: str): + if verbose: + print(msg % (arg1, arg2, arg3)) + + +@dispatch(str, str, str, str, str) +def v_print(msg: str, arg1: str, arg2: str, arg3: str, arg4: str): + if verbose: + print(msg % (arg1, arg2, arg3, arg4)) + + +@dispatch(str, int) +def v_print(msg: str, arg: int): + if verbose: + print(msg % int(arg)) + + +@dispatch(str, int, str) +def v_print(msg: str, arg1: int, arg2: str): + if verbose: + print(msg % (int(arg1), str(arg2))) + + +@dispatch(str, str, int) +def v_print(msg: str, arg1: str, arg2: int): + if verbose: + print(msg % (arg1, int(arg2))) + + +@dispatch(str, int, int) +def v_print(msg: str, arg1: int, arg2: int): + if verbose: + print(msg % (int(arg1), int(arg2))) + + +@dispatch(str, int, int, str) +def v_print(msg: str, arg1: int, arg2: int, arg3: str): + if verbose: + print(msg % (int(arg1), int(arg2), str(arg3))) + + +@dispatch(str, int, int, int) +def v_print(msg: str, arg1: int, arg2: int, arg3: int): + if verbose: + print(msg % (int(arg1), int(arg2), int(arg3))) + + +@dispatch(str, int, int, int, int) +def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): + if verbose: + print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) + + +def restful_execute(host: str, port: int, user: str, password: str, cmd: str): + url = "http://%s:%d/rest/sql" % (host, restPort) + + v_print("restful_execute - cmd: %s", cmd) + + resp = requests.post(url, cmd, auth=(user, password)) + + v_print("resp status: %d", resp.status_code) + + if debug: + v_print( + "resp text: %s", + json.dumps( + resp.json(), + sort_keys=True, + indent=2)) + else: + print("resp: %s" % json.dumps(resp.json())) + + +def query_func(process: int, thread: int, cmd: str): + v_print("%d process %d thread cmd: %s", process, thread, cmd) + + if oneMoreHost != "NotSupported" and random.randint( + 0, 1) == 1: + v_print("%s", "Send to second host") + if native: + cursor2.execute(cmd) + else: + restful_execute( + oneMoreHost, port, user, password, cmd) + else: + v_print("%s%s%s", "Send ", cmd, " to the host") + if native: + pass +# cursor.execute(cmd) + else: + restful_execute( + host, port, user, password, cmd) + + +def query_data_process(cmd: str): + # establish connection if native + if native: + v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + try: + conn = taos.connect( + host=host, + user=user, + password=password, + config=configDir) + v_print("conn: %s", str(conn.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + sys.exit(1) + + try: + cursor = conn.cursor() + v_print("cursor:%d %s", id(cursor), str(cursor.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + conn.close() + sys.exit(1) + + if native: + try: + cursor.execute(cmd) + cols = cursor.description + data = cursor.fetchall() + + for col in data: + print(col) + except Exception as e: + conn.close() + print("Error: %s" % e.args[0]) + sys.exit(1) + + else: + restful_execute( + host, + port, + user, + password, + cmd) + + if native: + cursor.close() + conn.close() + + +def create_stb(): + for i in range(0, numOfStb): + if native: + cursor.execute( + "CREATE TABLE IF NOT EXISTS %s%d (ts timestamp, value float) TAGS (uuid binary(50))" % + (stbName, i)) + else: + restful_execute( + host, + port, + user, + password, + "CREATE TABLE IF NOT EXISTS %s%d (ts timestamp, value float) TAGS (uuid binary(50))" % + (stbName, i) + ) + + +def use_database(): + + if native: + cursor.execute("USE %s" % current_db) + else: + restful_execute(host, port, user, password, "USE %s" % current_db) + + +def create_databases(): + for i in range(0, numOfDb): + v_print("will create database db%d", int(i)) + + if native: + cursor.execute( + "CREATE DATABASE IF NOT EXISTS %s%d" % (dbName, i)) + else: + restful_execute( + host, + port, + user, + password, + "CREATE DATABASE IF NOT EXISTS %s%d" % (dbName, i)) + + +def drop_tables(): + # TODO + v_print("TODO: drop tables total %d", numOfTb) + pass + + +def drop_stable(): + # TODO + v_print("TODO: drop stables total %d", numOfStb) + pass + + +def drop_databases(): + v_print("drop databases total %d", numOfDb) + + # drop exist databases first + for i in range(0, numOfDb): + v_print("will drop database db%d", int(i)) + + if native: + cursor.execute( + "DROP DATABASE IF EXISTS %s%d" % + (dbName, i)) + else: + restful_execute( + host, + port, + user, + password, + "DROP DATABASE IF EXISTS %s%d" % + (dbName, i)) + + +def insert_func(process: int, thread: int): + v_print("%d process %d thread, insert_func ", process, thread) + + # generate uuid + uuid_int = random.randint(0, numOfTb + 1) + uuid = "%s" % uuid_int + v_print("uuid is: %s", uuid) + + # establish connection if native + if native: + v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + try: + conn = taos.connect( + host=host, + user=user, + password=password, + config=configDir) + v_print("conn: %s", str(conn.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + sys.exit(1) + + try: + cursor = conn.cursor() + v_print("cursor:%d %s", id(cursor), str(cursor.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + conn.close() + sys.exit(1) + + v_print("numOfRec %d:", numOfRec) + + row = 0 + while row < numOfRec: + v_print("row: %d", row) + sqlCmd = ['INSERT INTO '] + try: + sqlCmd.append( + "%s.%s%d " % (current_db, tbName, thread)) + + if (numOfStb > 0 and autosubtable): + sqlCmd.append("USING %s.%s%d TAGS('%s') " % + (current_db, stbName, numOfStb - 1, uuid)) + + start_time = datetime.datetime( + 2021, 1, 25) + datetime.timedelta(seconds=row) + + sqlCmd.append("VALUES ") + for batchIter in range(0, batch): + sqlCmd.append("('%s', %f) " % + ( + start_time + + datetime.timedelta( + milliseconds=batchIter), + random.random())) + row = row + 1 + if row >= numOfRec: + v_print("BREAK, row: %d numOfRec:%d", row, numOfRec) + break + + except Exception as e: + print("Error: %s" % e.args[0]) + + cmd = ' '.join(sqlCmd) + + if measure: + exec_start_time = datetime.datetime.now() + + if native: + affectedRows = cursor.execute(cmd) + else: + restful_execute( + host, port, user, password, cmd) + + if measure: + exec_end_time = datetime.datetime.now() + exec_delta = exec_end_time - exec_start_time + v_print( + "consume %d microseconds", + exec_delta.microseconds) + + v_print("cmd: %s, length:%d", cmd, len(cmd)) + + if native: + cursor.close() + conn.close() + + +def create_tb_using_stb(): + # TODO: + pass + + +def create_tb(): + v_print("create_tb() numOfTb: %d", numOfTb) + for i in range(0, numOfDb): + if native: + cursor.execute("USE %s%d" % (dbName, i)) + else: + restful_execute( + host, port, user, password, "USE %s%d" % + (dbName, i)) + + for j in range(0, numOfTb): + if native: + cursor.execute( + "CREATE TABLE %s%d (ts timestamp, value float)" % + (tbName, j)) + else: + restful_execute( + host, + port, + user, + password, + "CREATE TABLE %s%d (ts timestamp, value float)" % + (tbName, j)) + + +def insert_data_process(lock, i: int, begin: int, end: int): + lock.acquire() + tasks = end - begin + v_print("insert_data_process:%d table from %d to %d, tasks %d", i, begin, end, tasks) + + if (threads < (end - begin)): + for j in range(begin, end, threads): + with ThreadPoolExecutor(max_workers=threads) as executor: + k = end if ((j + threads) > end) else (j + threads) + workers = [ + executor.submit( + insert_func, + i, + n) for n in range( + j, + k)] + wait(workers, return_when=ALL_COMPLETED) + else: + with ThreadPoolExecutor(max_workers=threads) as executor: + workers = [ + executor.submit( + insert_func, + i, + j) for j in range( + begin, + end)] + wait(workers, return_when=ALL_COMPLETED) + + lock.release() + + +def query_db(i): + if native: + cursor.execute("USE %s%d" % (dbName, i)) + else: + restful_execute( + host, port, user, password, "USE %s%d" % + (dbName, i)) + + for j in range(0, numOfTb): + if native: + cursor.execute( + "SELECT COUNT(*) FROM %s%d" % (tbName, j)) + else: + restful_execute( + host, port, user, password, "SELECT COUNT(*) FROM %s%d" % + (tbName, j)) + + +def printConfig(): + + print("###################################################################") + print("# Use native interface: %s" % native) + print("# Server IP: %s" % host) + if native: + print("# Server port: %s" % port) + else: + print("# Server port: %s" % restPort) + + print("# Configuration Dir: %s" % configDir) + print("# User: %s" % user) + print("# Password: %s" % password) + print("# Number of Columns per record: %s" % colsPerRecord) + print("# Number of Threads: %s" % threads) + print("# Number of Processes: %s" % processes) + print("# Number of Tables: %s" % numOfTb) + print("# Number of records per Table: %s" % numOfRec) + print("# Records/Request: %s" % batch) + print("# Database name: %s" % dbName) + print("# Replica: %s" % replica) + print("# Use STable: %s" % useStable) + print("# Table prefix: %s" % tbName) + if useStable: + print("# STable prefix: %s" % stbName) + + print("# Data order: %s" % outOfOrder) + print("# Data out of order rate: %s" % rateOOOO) + print("# Delete method: %s" % deleteMethod) + print("# Query command: %s" % queryCmd) + print("# Insert Only: %s" % insertOnly) + print("# Verbose output %s" % verbose) + print("# Test time: %s" % + datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S")) + print("###################################################################") + + +if __name__ == "__main__": + + native = False + verbose = False + debug = False + measure = True + dropDbOnly = False + colsPerRecord = 3 + numOfDb = 1 + dbName = "test" + replica = 1 + batch = 1 + numOfTb = 1 + tbName = "tb" + useStable = False + numOfStb = 0 + stbName = "stb" + numOfRec = 10 + ieration = 1 + host = "127.0.0.1" + configDir = "/etc/taos" + oneMoreHost = "NotSupported" + port = 6030 + restPort = 6041 + user = "root" + defaultPass = "taosdata" + processes = 1 + threads = 1 + insertOnly = False + autosubtable = False + queryCmd = "NO" + outOfOrder = 0 + rateOOOO = 0 + deleteMethod = 0 + skipPrompt = False + + try: + opts, args = getopt.gnu_getopt(sys.argv[1:], + 'Nh:p:u:P:d:a:m:Ms:Q:T:C:r:l:t:n:c:xOR:D:vgyH', + [ + 'native', 'host', 'port', 'user', 'password', 'dbname', 'replica', 'tbname', + 'stable', 'stbname', 'query', 'threads', 'processes', + 'recPerReq', 'colsPerRecord', 'numOfTb', 'numOfRec', 'config', + 'insertOnly', 'outOfOrder', 'rateOOOO', 'deleteMethod', + 'verbose', 'debug', 'skipPrompt', 'help' + ]) + except getopt.GetoptError as err: + print('ERROR:', err) + print('Try `taosdemo.py --help` for more options.') + sys.exit(1) + + if bool(opts) is False: + print('Try `taosdemo.py --help` for more options.') + sys.exit(1) + + for key, value in opts: + if key in ['-H', '--help']: + print('') + print( + 'taosdemo.py for TDengine') + print('') + print('Author: Shuduo Sang ') + print('') + + print('\t-H, --help Show usage.') + print('') + + print('\t-N, --native flag, Use native interface if set. Default is using RESTful interface.') + print('\t-h, --host host, The host to connect to TDengine. Default is localhost.') + print('\t-p, --port port, The TCP/IP port number to use for the connection. Default is 0.') + print('\t-u, --user user, The user name to use when connecting to the server. Default is \'root\'.') + print('\t-P, --password password, The password to use when connecting to the server. Default is \'taosdata\'.') + print('\t-l, --colsPerRec num_of_columns_per_record, The number of columns per record. Default is 3.') + print( + '\t-d, --dbname database, Destination database. Default is \'test\'.') + print('\t-a, --replica replica, Set the replica parameters of the database, Default 1, min: 1, max: 5.') + print( + '\t-m, --tbname
    table_prefix, Table prefix name. Default is \'t\'.') + print( + '\t-M, --stable flag, Use super table. Default is no') + print( + '\t-s, --stbname stable_prefix, STable prefix name. Default is \'st\'') + print('\t-Q, --query [NO|EACHTB|command] query, Execute query command. set \'EACHTB\' means select * from each table') + print( + '\t-T, --threads num_of_threads, The number of threads. Default is 1.') + print( + '\t-C, --processes num_of_processes, The number of threads. Default is 1.') + print('\t-r, --batch num_of_records_per_req, The number of records per request. Default is 1000.') + print( + '\t-t, --numOfTb num_of_tables, The number of tables. Default is 1.') + print('\t-n, --numOfRec num_of_records_per_table, The number of records per table. Default is 1.') + print('\t-c, --config config_directory, Configuration directory. Default is \'/etc/taos/\'.') + print('\t-x, --inserOnly flag, Insert only flag.') + print('\t-O, --outOfOrder out of order data insert, 0: In order, 1: Out of order. Default is in order.') + print('\t-R, --rateOOOO rate, Out of order data\'s rate--if order=1 Default 10, min: 0, max: 50.') + print('\t-D, --deleteMethod Delete data methods 0: don\'t delete, 1: delete by table, 2: delete by stable, 3: delete by database.') + print('\t-v, --verbose Print verbose output') + print('\t-g, --debug Print debug output') + print( + '\t-y, --skipPrompt Skip read key for continous test, default is not skip') + print('') + sys.exit(0) + + if key in ['-N', '--native']: + try: + import taos + except Exception as e: + print("Error: %s" % e.args[0]) + sys.exit(1) + native = True + + if key in ['-h', '--host']: + host = value + + if key in ['-p', '--port']: + port = int(value) + + if key in ['-u', '--user']: + user = value + + if key in ['-P', '--password']: + password = value + else: + password = defaultPass + + if key in ['-d', '--dbname']: + dbName = value + + if key in ['-a', '--replica']: + replica = int(value) + if replica < 1: + print("FATAL: number of replica need > 0") + sys.exit(1) + + if key in ['-m', '--tbname']: + tbName = value + + if key in ['-M', '--stable']: + useStable = True + numOfStb = 1 + + if key in ['-s', '--stbname']: + stbName = value + + if key in ['-Q', '--query']: + queryCmd = str(value) + + if key in ['-T', '--threads']: + threads = int(value) + if threads < 1: + print("FATAL: number of threads must be larger than 0") + sys.exit(1) + + if key in ['-C', '--processes']: + processes = int(value) + if processes < 1: + print("FATAL: number of processes must be larger than 0") + sys.exit(1) + + if key in ['-r', '--batch']: + batch = int(value) + + if key in ['-l', '--colsPerRec']: + colsPerRec = int(value) + + if key in ['-t', '--numOfTb']: + numOfTb = int(value) + v_print("numOfTb is %d", numOfTb) + + if key in ['-n', '--numOfRec']: + numOfRec = int(value) + v_print("numOfRec is %d", numOfRec) + if numOfRec < 1: + print("FATAL: number of records must be larger than 0") + sys.exit(1) + + + if key in ['-c', '--config']: + configDir = value + v_print("config dir: %s", configDir) + + if key in ['-x', '--insertOnly']: + insertOnly = True + v_print("insert only: %d", insertOnly) + + if key in ['-O', '--outOfOrder']: + outOfOrder = int(value) + v_print("out of order is %d", outOfOrder) + + if key in ['-R', '--rateOOOO']: + rateOOOO = int(value) + v_print("the rate of out of order is %d", rateOOOO) + + if key in ['-D', '--deleteMethod']: + deleteMethod = int(value) + if (deleteMethod < 0) or (deleteMethod > 3): + print( + "inputed delete method is %d, valid value is 0~3, set to default 0" % + deleteMethod) + deleteMethod = 0 + v_print("the delete method is %d", deleteMethod) + + if key in ['-v', '--verbose']: + verbose = True + + if key in ['-g', '--debug']: + debug = True + + if key in ['-y', '--skipPrompt']: + skipPrompt = True + + if verbose: + printConfig() + + if not skipPrompt: + input("Press any key to continue..") + + # establish connection first if native + if native: + v_print("host:%s, user:%s passwd:%s configDir:%s ", host, user, password, configDir) + try: + conn = taos.connect( + host=host, + user=user, + password=password, + config=configDir) + v_print("conn: %s", str(conn.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + sys.exit(1) + + try: + cursor = conn.cursor() + v_print("cursor:%d %s", id(cursor), str(cursor.__class__)) + except Exception as e: + print("Error: %s" % e.args[0]) + conn.close() + sys.exit(1) + + # drop data only if delete method be set + if deleteMethod > 0: + if deleteMethod == 1: + drop_tables() + print("Drop tables done.") + elif deleteMethod == 2: + drop_stables() + print("Drop super tables done.") + elif deleteMethod == 3: + drop_databases() + print("Drop Database done.") + sys.exit(0) + + # create databases + drop_databases() + create_databases() + + # use last database + current_db = "%s%d" % (dbName, (numOfDb - 1)) + use_database() + + if measure: + start_time_begin = time.time() + + if numOfStb > 0: + create_stb() + if (autosubtable == False): + create_tb_using_stb() + else: + create_tb() + + if measure: + end_time = time.time() + print( + "Total time consumed {} seconds for create table.".format( + (end_time - start_time_begin))) + + if native: + cursor.close() + conn.close() + + # start insert data + if measure: + start_time = time.time() + + manager = Manager() + lock = manager.Lock() + pool = Pool(processes) + + begin = 0 + end = 0 + + quotient = numOfTb // processes + if quotient < 1: + processes = numOfTb + quotient = 1 + + remainder = numOfTb % processes + v_print( + "num of tables: %d, quotient: %d, remainder: %d", + numOfTb, + quotient, + remainder) + + for i in range(processes): + begin = end + + if i < remainder: + end = begin + quotient + 1 + else: + end = begin + quotient + pool.apply_async(insert_data_process, args=(lock, i, begin, end,)) + + pool.close() + pool.join() + time.sleep(1) + + if measure: + end_time = time.time() + print( + "Total time consumed {} seconds for insert data.".format( + (end_time - start_time))) + + + # query data + if queryCmd != "NO": + print("queryCmd: %s" % queryCmd) + query_data_process(queryCmd) + + if measure: + end_time = time.time() + print( + "Total time consumed {} seconds.".format( + (end_time - start_time_begin))) + + print("done") diff --git a/tests/examples/rust b/tests/examples/rust new file mode 160000 index 0000000000000000000000000000000000000000..1c8924dc668e6aa848214c2fc54e3ace3f5bf8df --- /dev/null +++ b/tests/examples/rust @@ -0,0 +1 @@ +Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df diff --git a/tests/perftest-scripts/coverage_test.sh b/tests/perftest-scripts/coverage_test.sh index 5085ec89d00def1a2147f24ccbeb488406541775..1be2053f24f28dd825c040912783675b1eab94f9 100755 --- a/tests/perftest-scripts/coverage_test.sh +++ b/tests/perftest-scripts/coverage_test.sh @@ -56,9 +56,9 @@ function runGeneralCaseOneByOne { case=`echo $line | grep sim$ |awk '{print $NF}'` if [ -n "$case" ]; then - ./test.sh -f $case > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT || \ - echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT + date +%F\ %T | tee -a $TDENGINE_COVERAGE_REPORT && ./test.sh -f $case > /dev/null 2>&1 && \ + ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a $TDENGINE_COVERAGE_REPORT ) \ + || echo -e "${RED}$case failed${NC}" | tee -a $TDENGINE_COVERAGE_REPORT fi fi done < $1 diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index 8498094181857095fa52123b364119801dc93d56..8b9fa1a546eedc955b4b902a20731a4736d8f60d 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -39,6 +39,8 @@ function buildTDengine { cd $WORK_DIR/TDengine git remote update > /dev/null + git reset --hard HEAD + git checkout develop REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` LOCAL_COMMIT=`git rev-parse --short @` @@ -48,47 +50,35 @@ function buildTDengine { echo "repo up-to-date" else echo "repo need to pull" - git pull > /dev/null + git pull > /dev/null 2>&1 LOCAL_COMMIT=`git rev-parse --short @` cd debug rm -rf * cmake .. > /dev/null - make > /dev/null - make install + make && make install > /dev/null fi } function runQueryPerfTest { [ -f $PERFORMANCE_TEST_REPORT ] && rm $PERFORMANCE_TEST_REPORT nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taosperf/ > /dev/null 2>&1 & - echoInfo "Run Performance Test" + echoInfo "Wait TDengine to start" + sleep 120 + echoInfo "Run Performance Test" cd $WORK_DIR/TDengine/tests/pytest python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - - yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -t 1000 -n 1000 > taosdemoperf.txt - - CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` - INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` - REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'` - AVGDELAY=`echo ${delay:0:${#delay}-3}` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'` - MAXDELAY=`echo ${delay:0:${#delay}-3}` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'` - MINDELAY=`echo ${delay:0:${#delay}-2}` - python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY | tee -a $PERFORMANCE_TEST_REPORT - [ -f taosdemoperf.txt ] && rm taosdemoperf.txt + python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT } function sendReport { echo "send report" - receiver="pxiao@taosdata.com" + receiver="develop@taosdata.com" mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" cd $TDENGINE_DIR @@ -104,6 +94,7 @@ function sendReport { stopTaosd buildTDengine runQueryPerfTest +stopTaosd echoInfo "Send Report" sendReport diff --git a/tests/perftest-scripts/perftest-taosdemo.sh b/tests/perftest-scripts/perftest-taosdemo.sh index ae9e8dd7371cca2f522810da6d7816adba86a862..dffa251ef76d866f33247bd0ce6fd5bb718449de 100755 --- a/tests/perftest-scripts/perftest-taosdemo.sh +++ b/tests/perftest-scripts/perftest-taosdemo.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes function red_echo { echo -e "\033[31m$@\033[0m"; } function green_echo { echo -e "\033[32m$@\033[0m"; } diff --git a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh index 110ab9e5fa6463db9f55c8320b1fd3e36380d7a1..3f56a048e9f78b9e6d02bf8f1d704d27d46e1149 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-13d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-13d.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # diff --git a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh index 9e8bc697bc18e0e5c3a6d5957fed948192c09241..29e72d7b3ff62aa30fdefd694e435335e752e630 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-1d.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-1d.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh index 5fc5b9d03a4339263eb07a9af9b76ee2fc324f1b..3bc63b831b901b9a8a4b9b96385574a345ab58c6 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int100s.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # diff --git a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh index bafa04f174955d9938810f9b5bc353a0b712c3c3..566479c967320e9bbb8a9e877d26e1bbeb750add 100755 --- a/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh +++ b/tests/perftest-scripts/perftest-tsdb-compare-var10k-int10s.sh @@ -16,6 +16,9 @@ fi logDir=`grep "^logDir" /etc/taos/taos.cfg | awk '{print $2}'` dataDir=`grep "^dataDir" /etc/taos/taos.cfg | awk '{print $2}'` +[ -z "$logDir" ] && logDir="/var/log/taos" +[ -z "$dataDir" ] && dataDir="/var/lib/taos" + # Coloured Echoes # function red_echo { echo -e "\033[31m$@\033[0m"; } # function green_echo { echo -e "\033[32m$@\033[0m"; } # diff --git a/tests/pytest/account/account_create.py b/tests/pytest/account/account_create.py index 85adfff19994a3fbb12611e28d9872fb6cb1aeca..c008acccd8d8d33af0ca77bd161d4ab020c1543e 100644 --- a/tests/pytest/account/account_create.py +++ b/tests/pytest/account/account_create.py @@ -24,6 +24,17 @@ class TDTestCase: tdSql.init(conn.cursor(), logSql) def run(self): + tdSql.query("show users") + rows = tdSql.queryRows + + tdSql.execute("create user test PASS 'test' ") + tdSql.query("show users") + tdSql.checkRows(rows + 1) + + tdSql.error("create user tdenginetdenginetdengine PASS 'test' ") + + tdSql.error("create user tdenginet PASS '1234512345123456' ") + try: tdSql.execute("create account a&cc PASS 'pass123'") except Exception as e: @@ -31,6 +42,7 @@ class TDTestCase: return tdLog.exit("drop built-in user is error.") + def stop(self): tdSql.close() diff --git a/tests/pytest/alter/alter_debugFlag.py b/tests/pytest/alter/alter_debugFlag.py new file mode 100644 index 0000000000000000000000000000000000000000..38d972b58252a704420af0dda7c091c22c894ecd --- /dev/null +++ b/tests/pytest/alter/alter_debugFlag.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import random +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + flagList=["debugflag", "cdebugflag", "tmrDebugFlag", "uDebugFlag", "rpcDebugFlag"] + + for flag in flagList: + tdSql.execute("alter local %s 131" % flag) + tdSql.execute("alter local %s 135" % flag) + tdSql.execute("alter local %s 143" % flag) + randomFlag = random.randint(100, 250) + if randomFlag != 131 and randomFlag != 135 and randomFlag != 143: + tdSql.error("alter local %s %d" % (flag, randomFlag)) + + tdSql.query("show dnodes") + dnodeId = tdSql.getData(0, 0) + + for flag in flagList: + tdSql.execute("alter dnode %d %s 131" % (dnodeId, flag)) + tdSql.execute("alter dnode %d %s 135" % (dnodeId, flag)) + tdSql.execute("alter dnode %d %s 143" % (dnodeId, flag)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/alter/alter_stable.py b/tests/pytest/alter/alter_stable.py index 6852f4ef4ab6730f6423ba388d4f088ca229312e..6dad43313248388ab34a51f6dd40414f67ebdb7f 100644 --- a/tests/pytest/alter/alter_stable.py +++ b/tests/pytest/alter/alter_stable.py @@ -19,6 +19,10 @@ class TDTestCase: "double", "smallint", "tinyint", + "int unsigned", + "bigint unsigned", + "smallint unsigned", + "tinyint unsigned", "binary(10)", "nchar(10)", "timestamp"] diff --git a/tests/pytest/alter/alter_table.py b/tests/pytest/alter/alter_table.py index 2982492f6567ad54a62a3fcca44d4f1ac9560697..a5acb7a73e0844ac2235590cbc8b7793ef40746c 100644 --- a/tests/pytest/alter/alter_table.py +++ b/tests/pytest/alter/alter_table.py @@ -19,6 +19,10 @@ class TDTestCase: "double", "smallint", "tinyint", + "int unsigned", + "bigint unsigned", + "smallint unsigned", + "tinyint unsigned", "binary(10)", "nchar(10)", "timestamp"] @@ -126,6 +130,7 @@ class TDTestCase: for i in range(2, size): tdSql.checkData(0, i, self.rowNum * (size - i)) + tdSql.execute("create table st(ts timestamp, c1 int) tags(t1 float)") tdSql.execute("create table t0 using st tags(null)") tdSql.execute("alter table t0 set tag t1=2.1") diff --git a/tests/pytest/alter/alter_table_crash.py b/tests/pytest/alter/alter_table_crash.py index d1af022e35e636b15bc6e0fa175c7c41ed1ff640..2ec24860db9c078445cd186807a4c1834469f909 100644 --- a/tests/pytest/alter/alter_table_crash.py +++ b/tests/pytest/alter/alter_table_crash.py @@ -66,6 +66,14 @@ class TDTestCase: "alter table dt add column tbcol8 nchar(20)") tdSql.execute( "alter table dt add column tbcol9 binary(20)") + tdSql.execute( + "alter table dt add column tbcol10 tinyint unsigned") + tdSql.execute( + "alter table dt add column tbcol11 int unsigned") + tdSql.execute( + "alter table dt add column tbcol12 smallint unsigned") + tdSql.execute( + "alter table dt add column tbcol13 bigint unsigned") # restart taosd tdDnodes.forcestop(1) diff --git a/tests/pytest/cluster/clusterEnvSetup/Dockerfile b/tests/pytest/cluster/clusterEnvSetup/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..437dbc65e6430deb20faa16fc78ddc07005c15ac --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:latest AS builder + +ARG PACKAGE=TDengine-server-1.6.5.10-Linux-x64.tar.gz +ARG EXTRACTDIR=TDengine-enterprise-server +ARG TARBITRATORPKG=TDengine-tarbitrator-1.6.5.10-Linux-x64.tar.gz +ARG EXTRACTDIR2=TDengine-enterprise-arbitrator +ARG CONTENT=taos.tar.gz + +WORKDIR /root + +COPY ${PACKAGE} . +COPY ${TARBITRATORPKG} . + +RUN tar -zxf ${PACKAGE} +RUN tar -zxf ${TARBITRATORPKG} +RUN mv ${EXTRACTDIR}/driver ./lib +RUN tar -zxf ${EXTRACTDIR}/${CONTENT} +RUN mv ${EXTRACTDIR2}/bin/* /root/bin + +FROM ubuntu:latest + +WORKDIR /root + +RUN apt-get update +RUN apt-get install -y vim tmux net-tools +RUN echo 'alias ll="ls -l --color=auto"' >> /root/.bashrc +RUN ulimit -c unlimited + +COPY --from=builder /root/bin/taosd /usr/bin +COPY --from=builder /root/bin/tarbitrator /usr/bin +COPY --from=builder /root/bin/taosdemo /usr/bin +COPY --from=builder /root/bin/taosdump /usr/bin +COPY --from=builder /root/bin/taos /usr/bin +COPY --from=builder /root/cfg/taos.cfg /etc/taos/ +COPY --from=builder /root/lib/libtaos.so.* /usr/lib/libtaos.so.1 + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LC_CTYPE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 + +EXPOSE 6030-6042/tcp 6060/tcp 6030-6039/udp + +# VOLUME [ "/var/lib/taos", "/var/log/taos", "/etc/taos" ] + +CMD [ "bash" ] \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/basic.py b/tests/pytest/cluster/clusterEnvSetup/basic.py new file mode 100644 index 0000000000000000000000000000000000000000..dc7e07fd5cea3473270afa6cb2ddbfba6c2402e4 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/basic.py @@ -0,0 +1,119 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import taos +import random +import argparse + +class BuildDockerCluser: + + def __init__(self, hostName, user, password, configDir, numOfNodes, clusterVersion, dockerDir, removeFlag): + self.hostName = hostName + self.user = user + self.password = password + self.configDir = configDir + self.numOfNodes = numOfNodes + self.clusterVersion = clusterVersion + self.dockerDir = dockerDir + self.removeFlag = removeFlag + + def getConnection(self): + self.conn = taos.connect( + host = self.hostName, + user = self.user, + password = self.password, + config = self.configDir) + + def createDondes(self): + self.cursor = self.conn.cursor() + for i in range(2, self.numOfNodes + 1): + self.cursor.execute("create dnode tdnode%d" % i) + + def startArbitrator(self): + print("start arbitrator") + os.system("docker exec -d $(docker ps|grep tdnode1|awk '{print $1}') tarbitrator") + + def run(self): + if self.numOfNodes < 2 or self.numOfNodes > 10: + print("the number of nodes must be between 2 and 10") + exit(0) + print("remove Flag value %s" % self.removeFlag) + if self.removeFlag == False: + os.system("./cleanClusterEnv.sh -d %s" % self.dockerDir) + os.system("./buildClusterEnv.sh -n %d -v %s -d %s" % (self.numOfNodes, self.clusterVersion, self.dockerDir)) + self.getConnection() + self.createDondes() + self.startArbitrator() + +parser = argparse.ArgumentParser() +parser.add_argument( + '-H', + '--host', + action='store', + default='tdnode1', + type=str, + help='host name to be connected (default: tdnode1)') +parser.add_argument( + '-u', + '--user', + action='store', + default='root', + type=str, + help='user (default: root)') +parser.add_argument( + '-p', + '--password', + action='store', + default='taosdata', + type=str, + help='password (default: taosdata)') +parser.add_argument( + '-c', + '--config-dir', + action='store', + default='/etc/taos', + type=str, + help='configuration directory (default: /etc/taos)') +parser.add_argument( + '-n', + '--num-of-nodes', + action='store', + default=2, + type=int, + help='number of nodes in the cluster (default: 2, min: 2, max: 5)') +parser.add_argument( + '-v', + '--version', + action='store', + default='2.0.18.1', + type=str, + help='the version of the cluster to be build, Default is 2.0.17.1') +parser.add_argument( + '-d', + '--docker-dir', + action='store', + default='/data', + type=str, + help='the data dir for docker, default is /data') +parser.add_argument( + '--flag', + action='store_true', + help='remove docker containers flag, default: True') + +args = parser.parse_args() +cluster = BuildDockerCluser(args.host, args.user, args.password, args.config_dir, args.num_of_nodes, args.version, args.docker_dir, args.flag) +cluster.run() + +# usage 1: python3 basic.py -n 2 --flag (flag is True) +# usage 2: python3 basic.py -n 2 (flag should be False when it is not specified) \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh new file mode 100755 index 0000000000000000000000000000000000000000..60c81cd82b916a290c190c44f7b96f53154c4731 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/buildClusterEnv.sh @@ -0,0 +1,159 @@ +#!/bin/bash +echo "Executing buildClusterEnv.sh" +CURR_DIR=`pwd` +IN_TDINTERNAL="community" + +if [ $# != 6 ]; then + echo "argument list need input : " + echo " -n numOfNodes" + echo " -v version" + echo " -d docker dir" + exit 1 +fi + +NUM_OF_NODES= +VERSION= +DOCKER_DIR= +while getopts "n:v:d:" arg +do + case $arg in + n) + NUM_OF_NODES=$OPTARG + ;; + v) + VERSION=$OPTARG + ;; + d) + DOCKER_DIR=$OPTARG + ;; + ?) + echo "unkonwn argument" + ;; + esac +done + +function addTaoscfg { + for((i=1;i<=$NUM_OF_NODES;i++)) + do + touch $DOCKER_DIR/node$i/cfg/taos.cfg + echo 'firstEp tdnode1:6030' > $DOCKER_DIR/node$i/cfg/taos.cfg + echo 'fqdn tdnode'$i >> $DOCKER_DIR/node$i/cfg/taos.cfg + echo 'arbitrator tdnode1:6042' >> $DOCKER_DIR/node$i/cfg/taos.cfg + done +} + +function createDIR { + for((i=1;i<=$NUM_OF_NODES;i++)) + do + mkdir -p $DOCKER_DIR/node$i/data + mkdir -p $DOCKER_DIR/node$i/log + mkdir -p $DOCKER_DIR/node$i/cfg + mkdir -p $DOCKER_DIR/node$i/core + done +} + +function cleanEnv { + echo "Clean up docker environment" + for((i=1;i<=$NUM_OF_NODES;i++)) + do + rm -rf $DOCKER_DIR/node$i/data/* + rm -rf $DOCKER_DIR/node$i/log/* + done +} + +function prepareBuild { + + if [ -d $CURR_DIR/../../../../release ]; then + echo release exists + rm -rf $CURR_DIR/../../../../release/* + fi + + cd $CURR_DIR/../../../../packaging + + if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + if [ ! -e $DOCKER_DIR/TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + + echo "generating TDeninge enterprise packages" + ./release.sh -v cluster -n $VERSION >> /dev/null 2>&1 + + if [ ! -e $CURR_DIR/../../../../release/TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz ]; then + echo "no TDengine install package found" + exit 1 + fi + + if [ ! -e $CURR_DIR/../../../../release/TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + echo "no arbitrator install package found" + exit 1 + fi + + cd $CURR_DIR/../../../../release + mv TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + mv TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + fi + else + if [ ! -e $DOCKER_DIR/TDengine-server-$VERSION-Linux-x64.tar.gz ] || [ ! -e $DOCKER_DIR/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + + echo "generating TDeninge community packages" + ./release.sh -v edge -n $VERSION >> /dev/null 2>&1 + + if [ ! -e $CURR_DIR/../../../../release/TDengine-server-$VERSION-Linux-x64.tar.gz ]; then + echo "no TDengine install package found" + exit 1 + fi + + if [ ! -e $CURR_DIR/../../../../release/TDengine-arbitrator-$VERSION-Linux-x64.tar.gz ]; then + echo "no arbitrator install package found" + exit 1 + fi + + cd $CURR_DIR/../../../../release + mv TDengine-server-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + mv TDengine-arbitrator-$VERSION-Linux-x64.tar.gz $DOCKER_DIR + fi + fi + + rm -rf $DOCKER_DIR/*.yml + cd $CURR_DIR + + cp *.yml $DOCKER_DIR + cp Dockerfile $DOCKER_DIR +} + +function clusterUp { + echo "docker compose start" + + cd $DOCKER_DIR + + if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + docker_run="PACKAGE=TDengine-enterprise-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-enterprise-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-enterprise-server-$VERSION DIR2=TDengine-enterprise-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " + else + docker_run="PACKAGE=TDengine-server-$VERSION-Linux-x64.tar.gz TARBITRATORPKG=TDengine-arbitrator-$VERSION-Linux-x64.tar.gz DIR=TDengine-server-$VERSION DIR2=TDengine-arbitrator-$VERSION VERSION=$VERSION DATADIR=$DOCKER_DIR docker-compose -f docker-compose.yml " + fi + + if [ $NUM_OF_NODES -ge 2 ];then + echo "create $NUM_OF_NODES dnodes" + for((i=3;i<=$NUM_OF_NODES;i++)) + do + if [ ! -f node$i.yml ];then + echo "node$i.yml not exist" + cp node3.yml node$i.yml + sed -i "s/td2.0-node3/td2.0-node$i/g" node$i.yml + sed -i "s/'tdnode3'/'tdnode$i'/g" node$i.yml + sed -i "s#/node3/#/node$i/#g" node$i.yml + sed -i "s#hostname: tdnode3#hostname: tdnode$i#g" node$i.yml + sed -i "s#ipv4_address: 172.27.0.9#ipv4_address: 172.27.0.`expr $i + 6`#g" node$i.yml + fi + docker_run=$docker_run" -f node$i.yml " + done + docker_run=$docker_run" up -d" + fi + echo $docker_run |sh + + echo "docker compose finish" +} + +createDIR +cleanEnv +addTaoscfg +prepareBuild +clusterUp \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/cleanClusterEnv.sh b/tests/pytest/cluster/clusterEnvSetup/cleanClusterEnv.sh new file mode 100755 index 0000000000000000000000000000000000000000..675cae5fef850947c2c53899ef1863dc00b55422 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/cleanClusterEnv.sh @@ -0,0 +1,39 @@ +#!/bin/bash +echo "Executing cleanClusterEnv.sh" +CURR_DIR=`pwd` + +if [ $# != 2 ]; then + echo "argument list need input : " + echo " -d docker dir" + exit 1 +fi + +DOCKER_DIR= +while getopts "d:" arg +do + case $arg in + d) + DOCKER_DIR=$OPTARG + ;; + ?) + echo "unkonwn argument" + ;; + esac +done + +function removeDockerContainers { + cd $DOCKER_DIR + docker-compose down --remove-orphans +} + +function cleanEnv { + echo "Clean up docker environment" + for i in {1..10} + do + rm -rf $DOCKER_DIR/node$i/data/* + rm -rf $DOCKER_DIR/node$i/log/* + done +} + +removeDockerContainers +cleanEnv \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..d241062a2dd50fa82f934d9e915cddeb74505248 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/docker-compose.yml @@ -0,0 +1,126 @@ +version: '3.7' + +services: + td2.0-node1: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} + - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} + - DATADIR=${DATADIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode1' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode2:172.27.0.8" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + - "tdnode6:172.27.0.12" + - "tdnode7:172.27.0.13" + - "tdnode8:172.27.0.14" + - "tdnode9:172.27.0.15" + - "tdnode10:172.27.0.16" + volumes: + # bind data directory + - type: bind + source: ${DATADIR}/node1/data + target: /var/lib/taos + # bind log directory + - type: bind + source: ${DATADIR}/node1/log + target: /var/log/taos + # bind configuration + - type: bind + source: ${DATADIR}/node1/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: ${DATADIR}/node1/core + target: /coredump + - type: bind + source: /data + target: /root + hostname: tdnode1 + networks: + taos_update_net: + ipv4_address: 172.27.0.7 + command: taosd + + td2.0-node2: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} + - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} + - DATADIR=${DATADIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode2' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode1:172.27.0.7" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + volumes: + # bind data directory + - type: bind + source: ${DATADIR}/node2/data + target: /var/lib/taos + # bind log directory + - type: bind + source: ${DATADIR}/node2/log + target: /var/log/taos + # bind configuration + - type: bind + source: ${DATADIR}/node2/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: ${DATADIR}/node2/core + target: /coredump + - type: bind + source: ${DATADIR} + target: /root + hostname: tdnode2 + networks: + taos_update_net: + ipv4_address: 172.27.0.8 + command: taosd + + +networks: + taos_update_net: +# external: true + ipam: + driver: default + config: + - subnet: "172.27.0.0/24" diff --git a/src/kit/taosdemox/insert.json b/tests/pytest/cluster/clusterEnvSetup/insert.json similarity index 95% rename from src/kit/taosdemox/insert.json rename to tests/pytest/cluster/clusterEnvSetup/insert.json index aa071c115d60d78797b4c36456adfacd0d345af7..4548ef74e6f2b56e3fdf78e46792065330ee359e 100644 --- a/src/kit/taosdemox/insert.json +++ b/tests/pytest/cluster/clusterEnvSetup/insert.json @@ -8,6 +8,7 @@ "thread_count": 4, "thread_count_create_tbl": 1, "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", "databases": [{ "dbinfo": { "name": "db", @@ -38,7 +39,7 @@ "insert_rows": 100000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 1, - "rows_per_tbl": 100, + "interlace_rows": 100, "max_sql_len": 1024000, "disorder_ratio": 0, "disorder_range": 1000, diff --git a/tests/pytest/cluster/clusterEnvSetup/node3.yml b/tests/pytest/cluster/clusterEnvSetup/node3.yml new file mode 100644 index 0000000000000000000000000000000000000000..18f1b37c1c029f50cc6e66e662ec2a42bc475c5e --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/node3.yml @@ -0,0 +1,62 @@ +version: '3.7' + +services: + td2.0-node3: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} + - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} + - DATADIR=${DATADIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode3' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode2:172.27.0.8" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + - "tdnode6:172.27.0.12" + - "tdnode7:172.27.0.13" + - "tdnode8:172.27.0.14" + - "tdnode9:172.27.0.15" + - "tdnode10:172.27.0.16" + volumes: + # bind data directory + - type: bind + source: ${DATADIR}/node3/data + target: /var/lib/taos + # bind log directory + - type: bind + source: ${DATADIR}/node3/log + target: /var/log/taos + # bind configuration + - type: bind + source: ${DATADIR}/node3/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: ${DATADIR}/node3/core + target: /coredump + - type: bind + source: ${DATADIR} + target: /root + hostname: tdnode3 + networks: + taos_update_net: + ipv4_address: 172.27.0.9 + command: taosd \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/node4.yml b/tests/pytest/cluster/clusterEnvSetup/node4.yml new file mode 100644 index 0000000000000000000000000000000000000000..f542c22c4537dd6dc2c529dff39ba93c5c0ec3a4 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/node4.yml @@ -0,0 +1,62 @@ +version: '3.7' + +services: + td2.0-node4: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} + - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} + - DATADIR=${DATADIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode4' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode2:172.27.0.8" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + - "tdnode6:172.27.0.12" + - "tdnode7:172.27.0.13" + - "tdnode8:172.27.0.14" + - "tdnode9:172.27.0.15" + - "tdnode10:172.27.0.16" + volumes: + # bind data directory + - type: bind + source: ${DATADIR}/node4/data + target: /var/lib/taos + # bind log directory + - type: bind + source: ${DATADIR}/node4/log + target: /var/log/taos + # bind configuration + - type: bind + source: ${DATADIR}/node4/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: ${DATADIR}/node4/core + target: /coredump + - type: bind + source: ${DATADIR} + target: /root + hostname: tdnode4 + networks: + taos_update_net: + ipv4_address: 172.27.0.10 + command: taosd \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/node5.yml b/tests/pytest/cluster/clusterEnvSetup/node5.yml new file mode 100644 index 0000000000000000000000000000000000000000..dd5941ac769273031c98bca3248adcf633240088 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/node5.yml @@ -0,0 +1,62 @@ +version: '3.7' + +services: + td2.0-node5: + build: + context: . + args: + - PACKAGE=${PACKAGE} + - TARBITRATORPKG=${TARBITRATORPKG} + - EXTRACTDIR=${DIR} + - EXTRACTDIR2=${DIR2} + - DATADIR=${DATADIR} + image: 'tdengine:${VERSION}' + container_name: 'tdnode5' + cap_add: + - ALL + stdin_open: true + tty: true + environment: + TZ: "Asia/Shanghai" + command: > + sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && + echo $TZ > /etc/timezone && + mkdir /coredump && + echo 'kernel.core_pattern=/coredump/core_%e_%p' >> /etc/sysctl.conf && + sysctl -p && + exec my-main-application" + extra_hosts: + - "tdnode2:172.27.0.8" + - "tdnode3:172.27.0.9" + - "tdnode4:172.27.0.10" + - "tdnode5:172.27.0.11" + - "tdnode6:172.27.0.12" + - "tdnode7:172.27.0.13" + - "tdnode8:172.27.0.14" + - "tdnode9:172.27.0.15" + - "tdnode10:172.27.0.16" + volumes: + # bind data directory + - type: bind + source: ${DATADIR}/node5/data + target: /var/lib/taos + # bind log directory + - type: bind + source: ${DATADIR}/node5/log + target: /var/log/taos + # bind configuration + - type: bind + source: ${DATADIR}/node5/cfg + target: /etc/taos + # bind core dump path + - type: bind + source: ${DATADIR}/node5/core + target: /coredump + - type: bind + source: ${DATADIR} + target: /root + hostname: tdnode5 + networks: + taos_update_net: + ipv4_address: 172.27.0.11 + command: taosd \ No newline at end of file diff --git a/tests/pytest/cluster/clusterEnvSetup/taosdemoWrapper.py b/tests/pytest/cluster/clusterEnvSetup/taosdemoWrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..457dd4ee5aa5919951adbcea834d34cd367d3080 --- /dev/null +++ b/tests/pytest/cluster/clusterEnvSetup/taosdemoWrapper.py @@ -0,0 +1,142 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import random +import argparse + +class taosdemoWrapper: + + def __init__(self, host, metadata, database, tables, threads, configDir, replica, + columnType, columnsPerTable, rowsPerTable, disorderRatio, disorderRange, charTypeLen): + self.host = host + self.metadata = metadata + self.database = database + self.tables = tables + self.threads = threads + self.configDir = configDir + self.replica = replica + self.columnType = columnType + self.columnsPerTable = columnsPerTable + self.rowsPerTable = rowsPerTable + self.disorderRatio = disorderRatio + self.disorderRange = disorderRange + self.charTypeLen = charTypeLen + + def run(self): + if self.metadata is None: + os.system("taosdemo -h %s -d %s -t %d -T %d -c %s -a %d -b %s -n %d -t %d -O %d -R %d -w %d -x -y" + % (self.host, self.database, self.tables, self.threads, self.configDir, self.replica, self.columnType, + self.rowsPerTable, self.disorderRatio, self.disorderRange, self.charTypeLen)) + else: + os.system("taosdemo -f %s" % self.metadata) + + +parser = argparse.ArgumentParser() +parser.add_argument( + '-H', + '--host-name', + action='store', + default='tdnode1', + type=str, + help='host name to be connected (default: tdnode1)') +parser.add_argument( + '-f', + '--metadata', + action='store', + default=None, + type=str, + help='The meta data to execution procedure, if use -f, all other options invalid, Default is NULL') +parser.add_argument( + '-d', + '--db-name', + action='store', + default='test', + type=str, + help='Database name to be created (default: test)') +parser.add_argument( + '-t', + '--num-of-tables', + action='store', + default=10, + type=int, + help='Number of tables (default: 10000)') +parser.add_argument( + '-T', + '--num-of-threads', + action='store', + default=10, + type=int, + help='Number of rest threads (default: 10)') +parser.add_argument( + '-c', + '--config-dir', + action='store', + default='/etc/taos/', + type=str, + help='Configuration directory. (Default is /etc/taos/)') +parser.add_argument( + '-a', + '--replica', + action='store', + default=100, + type=int, + help='Set the replica parameters of the database (default: 1, min: 1, max: 3)') +parser.add_argument( + '-b', + '--column-type', + action='store', + default='int', + type=str, + help='the data_type of columns (default: TINYINT,SMALLINT,INT,BIGINT,FLOAT,DOUBLE,BINARY,NCHAR,BOOL,TIMESTAMP)') +parser.add_argument( + '-l', + '--num-of-cols', + action='store', + default=10, + type=int, + help='The number of columns per record (default: 10)') +parser.add_argument( + '-n', + '--num-of-rows', + action='store', + default=1000, + type=int, + help='Number of subtales per stable (default: 1000)') +parser.add_argument( + '-O', + '--disorder-ratio', + action='store', + default=0, + type=int, + help=' (0: in order, > 0: disorder ratio, default: 0)') +parser.add_argument( + '-R', + '--disorder-range', + action='store', + default=0, + type=int, + help='Out of order datas range, ms (default: 1000)') +parser.add_argument( + '-w', + '--char-type-length', + action='store', + default=16, + type=int, + help='Out of order datas range, ms (default: 16)') + +args = parser.parse_args() +taosdemo = taosdemoWrapper(args.host_name, args.metadata, args.db_name, args.num_of_tables, + args.num_of_threads, args.config_dir, args.replica, args.column_type, args.num_of_cols, + args.num_of_rows, args.disorder_ratio, args.disorder_range, args.char_type_length) +taosdemo.run() diff --git a/tests/pytest/cluster/clusterSetup.py b/tests/pytest/cluster/clusterSetup.py index dbda5657b6528402cc898762cccc78d74a196cd8..8a264270219232447a598768b45a37779edba698 100644 --- a/tests/pytest/cluster/clusterSetup.py +++ b/tests/pytest/cluster/clusterSetup.py @@ -11,15 +11,9 @@ # -*- coding: utf-8 -*- -import os -import sys -sys.path.insert(0, os.getcwd()) from fabric import Connection -from util.sql import * -from util.log import * -import taos import random -import threading +import time import logging class Node: @@ -76,6 +70,19 @@ class Node: print("remove taosd error for node %d " % self.index) logging.exception(e) + def forceStopOneTaosd(self): + try: + self.conn.run("kill -9 $(ps -ax|grep taosd|awk '{print $1}')") + except Exception as e: + print("kill taosd error on node%d " % self.index) + + def startOneTaosd(self): + try: + self.conn.run("nohup taosd -c /etc/taos/ > /dev/null 2>&1 &") + except Exception as e: + print("start taosd error on node%d " % self.index) + logging.exception(e) + def installTaosd(self, packagePath): self.conn.put(packagePath, self.homeDir) self.conn.cd(self.homeDir) @@ -122,100 +129,51 @@ class Node: class Nodes: def __init__(self): - self.node1 = Node(1, 'root', '52.151.60.239', 'node1', 'r', '/root/') - self.node2 = Node(2, 'root', '52.183.32.246', 'node1', 'r', '/root/') - self.node3 = Node(3, 'root', '51.143.46.79', 'node1', 'r', '/root/') - self.node4 = Node(4, 'root', '52.183.2.76', 'node1', 'r', '/root/') - self.node5 = Node(5, 'root', '13.66.225.87', 'node1', 'r', '/root/') + self.tdnodes = [] + self.tdnodes.append(Node(0, 'root', '52.143.103.7', 'node1', 'a', '/root/')) + self.tdnodes.append(Node(1, 'root', '52.250.48.222', 'node2', 'a', '/root/')) + self.tdnodes.append(Node(2, 'root', '51.141.167.23', 'node3', 'a', '/root/')) + self.tdnodes.append(Node(3, 'root', '52.247.207.173', 'node4', 'a', '/root/')) + self.tdnodes.append(Node(4, 'root', '51.141.166.100', 'node5', 'a', '/root/')) + + def stopOneNode(self, index): + self.tdnodes[index].forceStopOneTaosd() + + def startOneNode(self, index): + self.tdnodes[index].startOneTaosd() def stopAllTaosd(self): - self.node1.stopTaosd() - self.node2.stopTaosd() - self.node3.stopTaosd() - + for i in range(len(self.tdnodes)): + self.tdnodes[i].stopTaosd() + def startAllTaosd(self): - self.node1.startTaosd() - self.node2.startTaosd() - self.node3.startTaosd() + for i in range(len(self.tdnodes)): + self.tdnodes[i].startTaosd() def restartAllTaosd(self): - self.node1.restartTaosd() - self.node2.restartTaosd() - self.node3.restartTaosd() + for i in range(len(self.tdnodes)): + self.tdnodes[i].restartTaosd() def addConfigs(self, configKey, configValue): - self.node1.configTaosd(configKey, configValue) - self.node2.configTaosd(configKey, configValue) - self.node3.configTaosd(configKey, configValue) + for i in range(len(self.tdnodes)): + self.tdnodes[i].configTaosd(configKey, configValue) - def removeConfigs(self, configKey, configValue): - self.node1.removeTaosConfig(configKey, configValue) - self.node2.removeTaosConfig(configKey, configValue) - self.node3.removeTaosConfig(configKey, configValue) + def removeConfigs(self, configKey, configValue): + for i in range(len(self.tdnodes)): + self.tdnodes[i].removeTaosConfig(configKey, configValue) def removeAllDataFiles(self): - self.node1.removeData() - self.node2.removeData() - self.node3.removeData() - -class ClusterTest: - def __init__(self, hostName): - self.host = hostName - self.user = "root" - self.password = "taosdata" - self.config = "/etc/taos" - self.dbName = "mytest" - self.stbName = "meters" - self.numberOfThreads = 20 - self.numberOfTables = 10000 - self.numberOfRecords = 1000 - self.tbPrefix = "t" - self.ts = 1538548685000 - self.repeat = 1 - - def connectDB(self): - self.conn = taos.connect( - host=self.host, - user=self.user, - password=self.password, - config=self.config) - - def createSTable(self, replica): - cursor = self.conn.cursor() - tdLog.info("drop database if exists %s" % self.dbName) - cursor.execute("drop database if exists %s" % self.dbName) - tdLog.info("create database %s replica %d" % (self.dbName, replica)) - cursor.execute("create database %s replica %d" % (self.dbName, replica)) - tdLog.info("use %s" % self.dbName) - cursor.execute("use %s" % self.dbName) - tdLog.info("drop table if exists %s" % self.stbName) - cursor.execute("drop table if exists %s" % self.stbName) - tdLog.info("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName) - cursor.execute("create table %s(ts timestamp, current float, voltage int, phase int) tags(id int)" % self.stbName) - cursor.close() - - def insertData(self, threadID): - print("Thread %d: starting" % threadID) - cursor = self.conn.cursor() - tablesPerThread = int(self.numberOfTables / self.numberOfThreads) - baseTableID = tablesPerThread * threadID - for i in range (tablesPerThread): - cursor.execute("create table %s%d using %s tags(%d)" % (self.tbPrefix, baseTableID + i, self.stbName, baseTableID + i)) - query = "insert into %s%d values" % (self.tbPrefix, baseTableID + i) - base = self.numberOfRecords * i - for j in range(self.numberOfRecords): - query += "(%d, %f, %d, %d)" % (self.ts + base + j, random.random(), random.randint(210, 230), random.randint(0, 10)) - cursor.execute(query) - cursor.close() - print("Thread %d: finishing" % threadID) - - def run(self): - threads = [] - tdLog.info("Inserting data") - for i in range(self.numberOfThreads): - thread = threading.Thread(target=self.insertData, args=(i,)) - threads.append(thread) - thread.start() - - for i in range(self.numberOfThreads): - threads[i].join() \ No newline at end of file + for i in range(len(self.tdnodes)): + self.tdnodes[i].removeData() + +# kill taosd randomly every 10 mins +nodes = Nodes() +loop = 0 +while True: + loop = loop + 1 + index = random.randint(0, 4) + print("loop: %d, kill taosd on node%d" %(loop, index)) + nodes.stopOneNode(index) + time.sleep(60) + nodes.startOneNode(index) + time.sleep(600) \ No newline at end of file diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 7bdab8bc6717905e0d81309b0c948d2f3108b55c..333c2a0a578f6f399e742b0967db0185a22b5fde 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -18,6 +18,8 @@ import time import random import requests import argparse +import datetime +import string from requests.auth import HTTPBasicAuth func_list=['avg','count','twa','sum','stddev','leastsquares','min', 'max','first','last','top','bottom','percentile','apercentile', @@ -31,14 +33,14 @@ condition_list=[ 'fill(null)' ] -where_list = ['_c0>now-10d',' <50'," like \'%a%\'"] +where_list = ['_c0>now-10d',' <50','like',' is null'] class ConcurrentInquiry: # def __init__(self,ts=1500000001000,host='127.0.0.1',user='root',password='taosdata',dbname='test', # stb_prefix='st',subtb_prefix='t',n_Therads=10,r_Therads=10,probabilities=0.05,loop=5, # stableNum = 2,subtableNum = 1000,insertRows = 100): def __init__(self,ts,host,user,password,dbname, stb_prefix,subtb_prefix,n_Therads,r_Therads,probabilities,loop, - stableNum ,subtableNum ,insertRows ,mix_table): + stableNum ,subtableNum ,insertRows ,mix_table, replay): self.n_numOfTherads = n_Therads self.r_numOfTherads = r_Therads self.ts=ts @@ -54,13 +56,16 @@ class ConcurrentInquiry: self.subtb_stru_list=[] self.stb_tag_list=[] self.subtb_tag_list=[] - self.probabilities = [probabilities,1-probabilities] - self.ifjoin = [0,1] + self.probabilities = [1-probabilities,probabilities] + self.ifjoin = [1,0] self.loop = loop self.stableNum = stableNum self.subtableNum = subtableNum self.insertRows = insertRows self.mix_table = mix_table + self.max_ts = datetime.datetime.now() + self.min_ts = datetime.datetime.now() - datetime.timedelta(days=5) + self.replay = replay def SetThreadsNum(self,num): self.numOfTherads=num @@ -103,6 +108,14 @@ class ConcurrentInquiry: self.subtb_stru_list.append(tb) self.subtb_tag_list.append(tag) + def get_timespan(self,cl): #获取时间跨度(仅第一个超级表) + sql = 'select first(_c0),last(_c0) from ' + self.dbname + '.' + self.stb_list[0] + ';' + print(sql) + cl.execute(sql) + for data in cl: + self.max_ts = data[1] + self.min_ts = data[0] + def get_full(self): #获取所有的表、表结构 host = self.host user = self.user @@ -118,6 +131,7 @@ class ConcurrentInquiry: self.r_subtb_list(cl,i) self.r_stb_stru(cl) self.r_subtb_stru(cl) + self.get_timespan(cl) cl.close() conn.close() @@ -127,9 +141,21 @@ class ConcurrentInquiry: for i in range(random.randint(0,len(tlist))): c = random.choice(where_list) if c == '_c0>now-10d': - l.append(c) + rdate = self.min_ts + (self.max_ts - self.min_ts)/10 * random.randint(-11,11) + conlist = ' _c0 ' + random.choice(['<','>','>=','<=','<>']) + "'" + str(rdate) + "'" + if self.random_pick(): + l.append(conlist) + else: l.append(c) + elif '<50' in c: + conlist = ' ' + random.choice(tlist) + random.choice(['<','>','>=','<=','<>']) + str(random.randrange(-100,100)) + l.append(conlist) + elif 'is null' in c: + conlist = ' ' + random.choice(tlist) + random.choice([' is null',' is not null']) + l.append(conlist) else: - l.append(random.choice(tlist)+c) + s_all = string.ascii_letters + conlist = ' ' + random.choice(tlist) + " like \'%" + random.choice(s_all) + "%\' " + l.append(conlist) return 'where '+random.choice([' and ',' or ']).join(l) def con_interval(self,tlist,col_list,tag_list): @@ -195,8 +221,10 @@ class ConcurrentInquiry: if bool(random.getrandbits(1)): pick_func+=alias sel_col_list.append(pick_func) - - sql=sql+','.join(sel_col_list) #select col & func + if col_rand == 0: + sql = sql + '*' + else: + sql=sql+','.join(sel_col_list) #select col & func if self.mix_table == 0: sql = sql + ' from '+random.choice(self.stb_list+self.subtb_list)+' ' elif self.mix_table == 1: @@ -262,7 +290,26 @@ class ConcurrentInquiry: else: sel_col_tag.append('t1.' + str(random.choice(col_list[0] + tag_list[0]))) sel_col_tag.append('t2.' + str(random.choice(col_list[1] + tag_list[1]))) - sql += ','.join(sel_col_tag) + sel_col_list = [] + random.shuffle(func_list) + if self.random_pick(): + loop = 0 + for i,j in zip(sel_col_tag,func_list): #决定每个被查询col的函数 + alias = ' as '+ 'taos%d ' % loop + loop += 1 + pick_func = '' + if j == 'leastsquares': + pick_func=j+'('+i+',1,1)' + elif j == 'top' or j == 'bottom' or j == 'percentile' or j == 'apercentile': + pick_func=j+'('+i+',1)' + else: + pick_func=j+'('+i+')' + if bool(random.getrandbits(1)): + pick_func+=alias + sel_col_list.append(pick_func) + sql += ','.join(sel_col_list) + else: + sql += ','.join(sel_col_tag) sql = sql + ' from '+ str(tbname[0]) +' t1,' + str(tbname[1]) + ' t2 ' #select col & func join_section = None @@ -274,7 +321,6 @@ class ConcurrentInquiry: else: temp = random.choices(col_intersection+tag_intersection) join_section = temp.pop() - print(random.choices(col_intersection)) sql += 'where t1._c0 = t2._c0 and ' + 't1.' + str(join_section) + '=t2.' + str(join_section) return sql @@ -304,18 +350,27 @@ class ConcurrentInquiry: cl.execute("create database if not exists %s;" %self.dbname) cl.execute("use %s" % self.dbname) for k in range(stableNum): - sql="create table %s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20)) \ - tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20))" % (self.stb_prefix+str(k)) + sql="create table %s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20),c11 int unsigned,c12 smallint unsigned,c13 tinyint unsigned,c14 bigint unsigned) \ + tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20), t11 int unsigned , t12 smallint unsigned , t13 tinyint unsigned , t14 bigint unsigned)" % (self.stb_prefix+str(k)) cl.execute(sql) for j in range(subtableNum): - sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s')" % \ - (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j)) + if j % 100 == 0: + sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \ + (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k)) + else: + sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \ + (self.subtb_prefix+str(k)+'_'+str(j),self.stb_prefix+str(k),j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j), j%43, j%23 , j%17 , j%3167) print(sql) cl.execute(sql) for i in range(insertRows): - ret = cl.execute( - "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % - (self.subtb_prefix+str(k)+'_'+str(j),t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i),'涛思'+str(i))) + if i % 100 == 0 : + ret = cl.execute( + "insert into %s values (%d , NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % + (self.subtb_prefix+str(k)+'_'+str(j), t0+i)) + else: + ret = cl.execute( + "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % + (self.subtb_prefix+str(k)+'_'+str(j), t0+i, i%100, i/2.0, i%41, i%51, i%53, i*1.0, i%2,'taos'+str(i),'涛思'+str(i), i%43, i%23 , i%17 , i%3167)) cl.close() conn.close() @@ -367,7 +422,7 @@ class ConcurrentInquiry: ) cl = conn.cursor() cl.execute("use %s;" % self.dbname) - + fo = open('bak_sql_n_%d'%threadID,'w+') print("Thread %d: starting" % threadID) loop = self.loop while loop: @@ -378,6 +433,7 @@ class ConcurrentInquiry: else: sql=self.gen_query_join() print("sql is ",sql) + fo.write(sql+'\n') start = time.time() cl.execute(sql) cl.fetchall() @@ -393,13 +449,49 @@ class ConcurrentInquiry: exit(-1) loop -= 1 if loop == 0: break - + fo.close() cl.close() conn.close() print("Thread %d: finishing" % threadID) + + def query_thread_nr(self,threadID): #使用原生python接口进行重放 + host = self.host + user = self.user + password = self.password + conn = taos.connect( + host, + user, + password, + ) + cl = conn.cursor() + cl.execute("use %s;" % self.dbname) + replay_sql = [] + with open('bak_sql_n_%d'%threadID,'r') as f: + replay_sql = f.readlines() + print("Replay Thread %d: starting" % threadID) + for sql in replay_sql: + try: + print("sql is ",sql) + start = time.time() + cl.execute(sql) + cl.fetchall() + end = time.time() + print("time cost :",end-start) + except Exception as e: + print('-'*40) + print( + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql),str(e))) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + cl.close() + conn.close() + print("Replay Thread %d: finishing" % threadID) def query_thread_r(self,threadID): #使用rest接口查询 print("Thread %d: starting" % threadID) + fo = open('bak_sql_r_%d'%threadID,'w+') loop = self.loop while loop: try: @@ -408,6 +500,7 @@ class ConcurrentInquiry: else: sql=self.gen_query_join() print("sql is ",sql) + fo.write(sql+'\n') start = time.time() self.rest_query(sql) end = time.time() @@ -422,20 +515,53 @@ class ConcurrentInquiry: exit(-1) loop -= 1 if loop == 0: break - - print("Thread %d: finishing" % threadID) + fo.close() + print("Thread %d: finishing" % threadID) + + def query_thread_rr(self,threadID): #使用rest接口重放 + print("Replay Thread %d: starting" % threadID) + replay_sql = [] + with open('bak_sql_r_%d'%threadID,'r') as f: + replay_sql = f.readlines() + + for sql in replay_sql: + try: + print("sql is ",sql) + start = time.time() + self.rest_query(sql) + end = time.time() + print("time cost :",end-start) + except Exception as e: + print('-'*40) + print( + "Failure thread%d, sql: %s \nexception: %s" % + (threadID, str(sql),str(e))) + err_uec='Unable to establish connection' + if err_uec in str(e) and loop >0: + exit(-1) + print("Replay Thread %d: finishing" % threadID) def run(self): print(self.n_numOfTherads,self.r_numOfTherads) threads = [] - for i in range(self.n_numOfTherads): - thread = threading.Thread(target=self.query_thread_n, args=(i,)) - threads.append(thread) - thread.start() - for i in range(self.r_numOfTherads): - thread = threading.Thread(target=self.query_thread_r, args=(i,)) - threads.append(thread) - thread.start() + if self.replay: #whether replay + for i in range(self.n_numOfTherads): + thread = threading.Thread(target=self.query_thread_nr, args=(i,)) + threads.append(thread) + thread.start() + for i in range(self.r_numOfTherads): + thread = threading.Thread(target=self.query_thread_rr, args=(i,)) + threads.append(thread) + thread.start() + else: + for i in range(self.n_numOfTherads): + thread = threading.Thread(target=self.query_thread_n, args=(i,)) + threads.append(thread) + thread.start() + for i in range(self.r_numOfTherads): + thread = threading.Thread(target=self.query_thread_r, args=(i,)) + threads.append(thread) + thread.start() parser = argparse.ArgumentParser() parser.add_argument( @@ -550,13 +676,20 @@ parser.add_argument( default=0, type=int, help='0:stable & substable ,1:subtable ,2:stable (default: 0)') +parser.add_argument( + '-R', + '--replay', + action='store', + default=0, + type=int, + help='0:not replay ,1:replay (default: 0)') args = parser.parse_args() q = ConcurrentInquiry( args.ts,args.host_name,args.user,args.password,args.db_name, args.stb_name_prefix,args.subtb_name_prefix,args.number_of_native_threads,args.number_of_rest_threads, args.probabilities,args.loop_per_thread,args.number_of_stables,args.number_of_tables ,args.number_of_records, - args.mix_stable_subtable ) + args.mix_stable_subtable, args.replay ) if args.create_table: q.gen_data() diff --git a/tests/pytest/concurrent_inquiry.sh b/tests/pytest/concurrent_inquiry.sh new file mode 100755 index 0000000000000000000000000000000000000000..f426fbbcec3070789209eb787dba61d95571f0e5 --- /dev/null +++ b/tests/pytest/concurrent_inquiry.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# This is the script for us to try to cause the TDengine server or client to crash +# +# PREPARATION +# +# 1. Build an compile the TDengine source code that comes with this script, in the same directory tree +# 2. Please follow the direction in our README.md, and build TDengine in the build/ directory +# 3. Adjust the configuration file if needed under build/test/cfg/taos.cfg +# 4. Run the TDengine server instance: cd build; ./build/bin/taosd -c test/cfg +# 5. Make sure you have a working Python3 environment: run /usr/bin/python3 --version, and you should get 3.6 or above +# 6. Make sure you have the proper Python packages: # sudo apt install python3-setuptools python3-pip python3-distutils +# +# RUNNING THIS SCRIPT +# +# This script assumes the source code directory is intact, and that the binaries has been built in the +# build/ directory, as such, will will load the Python libraries in the directory tree, and also load +# the TDengine client shared library (so) file, in the build/directory, as evidenced in the env +# variables below. +# +# Running the script is simple, no parameter is needed (for now, but will change in the future). +# +# Happy Crashing... + + +# Due to the heavy path name assumptions/usage, let us require that the user be in the current directory +EXEC_DIR=`dirname "$0"` +if [[ $EXEC_DIR != "." ]] +then + echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)" + exit -1 +fi + +CURR_DIR=`pwd` +IN_TDINTERNAL="community" +if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + TAOS_DIR=$CURR_DIR/../../.. + TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` + LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6,7|rev`/lib +else + TAOS_DIR=$CURR_DIR/../.. + TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` + LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib +fi + +# Now getting ready to execute Python +# The following is the default of our standard dev env (Ubuntu 20.04), modify/adjust at your own risk +PYTHON_EXEC=python3.8 + +# First we need to set up a path for Python to find our own TAOS modules, so that "import" can work. +export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3:$(pwd) + +# Then let us set up the library path so that our compiled SO file can be loaded by Python +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR + +# Now we are all let, and let's see if we can find a crash. Note we pass all params +CONCURRENT_INQUIRY=concurrent_inquiry.py +if [[ $1 == '--valgrind' ]]; then + shift + export PYTHONMALLOC=malloc + VALGRIND_OUT=valgrind.out + VALGRIND_ERR=valgrind.err + # How to generate valgrind suppression file: https://stackoverflow.com/questions/17159578/generating-suppressions-for-memory-leaks + # valgrind --leak-check=full --gen-suppressions=all --log-fd=9 python3.8 ./concurrent_inquiry.py $@ 9>>memcheck.log + echo Executing under VALGRIND, with STDOUT/ERR going to $VALGRIND_OUT and $VALGRIND_ERR, please watch them from a different terminal. + valgrind \ + --leak-check=yes \ + --suppressions=crash_gen/valgrind_taos.supp \ + $PYTHON_EXEC \ + $CONCURRENT_INQUIRY $@ > $VALGRIND_OUT 2> $VALGRIND_ERR +elif [[ $1 == '--helgrind' ]]; then + shift + HELGRIND_OUT=helgrind.out + HELGRIND_ERR=helgrind.err + valgrind \ + --tool=helgrind \ + $PYTHON_EXEC \ + $CONCURRENT_INQUIRY $@ > $HELGRIND_OUT 2> $HELGRIND_ERR +else + $PYTHON_EXEC $CONCURRENT_INQUIRY $@ +fi + diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index c6b857b097671b8faf650f4659c2b2990ef67d97..0fe94bad1d6b36aa1be5d47428dae19e918ce6e4 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -35,16 +35,19 @@ import os import signal import traceback import resource -from guppy import hpy +# from guppy import hpy import gc from crash_gen.service_manager import ServiceManager, TdeInstance from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager +import crash_gen.settings import taos import requests +crash_gen.settings.init() + # Require Python 3 if sys.version_info[0] < 3: raise Exception("Must be using Python 3") @@ -259,6 +262,7 @@ class ThreadCoordinator: self._execStats = ExecutionStats() self._runStatus = Status.STATUS_RUNNING self._initDbs() + self._stepStartTime = None # Track how long it takes to execute each step def getTaskExecutor(self): return self._te @@ -354,10 +358,11 @@ class ThreadCoordinator: # end, and maybe signal them to stop if isinstance(err, CrashGenError): # our own transition failure Logging.info("State transition error") + # TODO: saw an error here once, let's print out stack info for err? traceback.print_stack() transitionFailed = True self._te = None # Not running any more - self._execStats.registerFailure("State transition error") + self._execStats.registerFailure("State transition error: {}".format(err)) else: raise # return transitionFailed # Why did we have this??!! @@ -393,6 +398,10 @@ class ThreadCoordinator: try: self._syncAtBarrier() # For now just cross the barrier Progress.emit(Progress.END_THREAD_STEP) + if self._stepStartTime : + stepExecTime = time.time() - self._stepStartTime + Progress.emitStr('{:.3f}s/{}'.format(stepExecTime, DbConnNative.totalRequests)) + DbConnNative.resetTotalRequests() # reset to zero except threading.BrokenBarrierError as err: self._execStats.registerFailure("Aborted due to worker thread timeout") Logging.error("\n") @@ -432,6 +441,7 @@ class ThreadCoordinator: # Then we move on to the next step Progress.emit(Progress.BEGIN_THREAD_STEP) + self._stepStartTime = time.time() self._releaseAllWorkerThreads(transitionFailed) if hasAbortedTask or transitionFailed : # abnormal ending, workers waiting at "gate" @@ -690,7 +700,7 @@ class AnyState: def canDropDb(self): # If user requests to run up to a number of DBs, # we'd then not do drop_db operations any more - if gConfig.max_dbs > 0 : + if gConfig.max_dbs > 0 or gConfig.use_shadow_db : return False return self._info[self.CAN_DROP_DB] @@ -698,6 +708,8 @@ class AnyState: return self._info[self.CAN_CREATE_FIXED_SUPER_TABLE] def canDropFixedSuperTable(self): + if gConfig.use_shadow_db: # duplicate writes to shaddow DB, in which case let's disable dropping s-table + return False return self._info[self.CAN_DROP_FIXED_SUPER_TABLE] def canAddData(self): @@ -882,8 +894,12 @@ class StateMechine: self._stateWeights = [1, 2, 10, 40] def init(self, dbc: DbConn): # late initailization, don't save the dbConn - self._curState = self._findCurrentState(dbc) # starting state - Logging.debug("Found Starting State: {}".format(self._curState)) + try: + self._curState = self._findCurrentState(dbc) # starting state + except taos.error.ProgrammingError as err: + Logging.error("Failed to initialized state machine, cannot find current state: {}".format(err)) + traceback.print_stack() + raise # re-throw # TODO: seems no lnoger used, remove? def getCurrentState(self): @@ -951,6 +967,8 @@ class StateMechine: # We transition the system to a new state by examining the current state itself def transition(self, tasks, dbc: DbConn): + global gSvcMgr + if (len(tasks) == 0): # before 1st step, or otherwise empty Logging.debug("[STT] Starting State: {}".format(self._curState)) return # do nothing @@ -1030,7 +1048,7 @@ class Database: _clsLock = threading.Lock() # class wide lock _lastInt = 101 # next one is initial integer _lastTick = 0 - _lastLaggingTick = 0 # lagging tick, for unsequenced insersions + _lastLaggingTick = 0 # lagging tick, for out-of-sequence (oos) data insertions def __init__(self, dbNum: int, dbc: DbConn): # TODO: remove dbc self._dbNum = dbNum # we assign a number to databases, for our testing purpose @@ -1086,21 +1104,24 @@ class Database: t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above - Logging.debug("Setting up TICKS to start from: {}".format(t4)) + Logging.info("Setting up TICKS to start from: {}".format(t4)) return t4 @classmethod - def getNextTick(cls): + def getNextTick(cls): + ''' + Fetch a timestamp tick, with some random factor, may not be unique. + ''' with cls._clsLock: # prevent duplicate tick if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized # 10k at 1/20 chance, should be enough to avoid overlaps tick = cls.setupLastTick() cls._lastTick = tick - cls._lastLaggingTick = tick + datetime.timedelta(0, -10000) + cls._lastLaggingTick = tick + datetime.timedelta(0, -60*2) # lagging behind 2 minutes, should catch up fast # if : # should be quite a bit into the future - if Dice.throw(20) == 0: # 1 in 20 chance, return lagging tick - cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds + if gConfig.mix_oos_data and Dice.throw(20) == 0: # if asked to do so, and 1 in 20 chance, return lagging tick + cls._lastLaggingTick += datetime.timedelta(0, 1) # pick the next sequence from the lagging tick sequence return cls._lastLaggingTick else: # regular # add one second to it @@ -1276,6 +1297,7 @@ class Task(): 0x510, # vnode not in ready state 0x14, # db not ready, errno changed 0x600, # Invalid table ID, why? + 0x218, # Table does not exist 1000 # REST catch-all error ]: return True # These are the ALWAYS-ACCEPTABLE ones @@ -1326,7 +1348,8 @@ class Task(): elif self._isErrAcceptable(errno2, err.__str__()): self.logDebug("[=] Acceptable Taos library exception: errno=0x{:X}, msg: {}, SQL: {}".format( errno2, err, wt.getDbConn().getLastSql())) - print("_", end="", flush=True) + # print("_", end="", flush=True) + Progress.emit(Progress.ACCEPTABLE_ERROR) self._err = err else: # not an acceptable error errMsg = "[=] Unexpected Taos library exception ({}): errno=0x{:X}, msg: {}, SQL: {}".format( @@ -1555,8 +1578,11 @@ class TaskCreateDb(StateTransitionTask): # numReplica = Dice.throw(gConfig.max_replicas) + 1 # 1,2 ... N numReplica = gConfig.max_replicas # fixed, always repStr = "replica {}".format(numReplica) - self.execWtSql(wt, "create database {} {}" - .format(self._db.getName(), repStr) ) + updatePostfix = "update 1" if gConfig.verify_data else "" # allow update only when "verify data" is active + dbName = self._db.getName() + self.execWtSql(wt, "create database {} {} {} ".format(dbName, repStr, updatePostfix ) ) + if dbName == "db_0" and gConfig.use_shadow_db: + self.execWtSql(wt, "create database {} {} {} ".format("db_s", repStr, updatePostfix ) ) class TaskDropDb(StateTransitionTask): @classmethod @@ -1759,20 +1785,20 @@ class TdSuperTable: 'top(speed, 50)', # TODO: not supported? 'bottom(speed, 50)', # TODO: not supported? 'apercentile(speed, 10)', # TODO: TD-1316 - 'last_row(speed)', + # 'last_row(speed)', # TODO: commented out per TD-3231, we should re-create # Transformation Functions # 'diff(speed)', # TODO: no supported?! 'spread(speed)' ]) # TODO: add more from 'top' - if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?! - sql = "select {} from {}.{}".format(aggExpr, self._dbName, self.getName()) - if Dice.throw(3) == 0: # 1 in X chance - sql = sql + ' GROUP BY color' - Progress.emit(Progress.QUERY_GROUP_BY) - # Logging.info("Executing GROUP-BY query: " + sql) - ret.append(SqlQuery(sql)) + # if aggExpr not in ['stddev(speed)']: # STDDEV not valid for super tables?! (Done in TD-1049) + sql = "select {} from {}.{}".format(aggExpr, self._dbName, self.getName()) + if Dice.throw(3) == 0: # 1 in X chance + sql = sql + ' GROUP BY color' + Progress.emit(Progress.QUERY_GROUP_BY) + # Logging.info("Executing GROUP-BY query: " + sql) + ret.append(SqlQuery(sql)) return ret @@ -1980,7 +2006,7 @@ class TaskAddData(StateTransitionTask): numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS fullTableName = db.getName() + '.' + regTableName - sql = "insert into {} values ".format(fullTableName) + sql = "INSERT INTO {} VALUES ".format(fullTableName) for j in range(numRecords): # number of records per table nextInt = db.getNextInt() nextTick = db.getNextTick() @@ -2008,12 +2034,24 @@ class TaskAddData(StateTransitionTask): # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written try: - sql = "insert into {} values ('{}', {}, '{}');".format( # removed: tags ('{}', {}) + sql = "INSERT INTO {} VALUES ('{}', {}, '{}');".format( # removed: tags ('{}', {}) fullTableName, # ds.getFixedSuperTableName(), # ds.getNextBinary(), ds.getNextFloat(), nextTick, nextInt, nextColor) dbc.execute(sql) + + # Quick hack, attach an update statement here. TODO: create an "update" task + if (not gConfig.use_shadow_db) and Dice.throw(5) == 0: # 1 in N chance, plus not using shaddow DB + nextInt = db.getNextInt() + nextColor = db.getNextColor() + sql = "INSERt INTO {} VALUES ('{}', {}, '{}');".format( # "INSERt" means "update" here + fullTableName, + nextTick, nextInt, nextColor) + # sql = "UPDATE {} set speed={}, color='{}' WHERE ts='{}'".format( + # fullTableName, db.getNextInt(), db.getNextColor(), nextTick) + dbc.execute(sql) + except: # Any exception at all if gConfig.verify_data: self.unlockTable(fullTableName) @@ -2062,7 +2100,8 @@ class TaskAddData(StateTransitionTask): random.shuffle(tblSeq) # now we have random sequence for i in tblSeq: if (i in self.activeTable): # wow already active - print("x", end="", flush=True) # concurrent insertion + # print("x", end="", flush=True) # concurrent insertion + Progress.emit(Progress.CONCURRENT_INSERTION) else: self.activeTable.add(i) # marking it active @@ -2365,11 +2404,16 @@ class MainExec: '--larger-data', action='store_true', help='Write larger amount of data during write operations (default: false)') + parser.add_argument( + '-m', + '--mix-oos-data', + action='store_false', + help='Mix out-of-sequence data into the test data stream (default: true)') parser.add_argument( '-n', '--dynamic-db-table-names', action='store_true', - help='Use non-fixed names for dbs/tables, useful for multi-instance executions (default: false)') + help='Use non-fixed names for dbs/tables, for -b, useful for multi-instance executions (default: false)') parser.add_argument( '-o', '--num-dnodes', @@ -2406,6 +2450,11 @@ class MainExec: '--verify-data', action='store_true', help='Verify data written in a number of places by reading back (default: false)') + parser.add_argument( + '-w', + '--use-shadow-db', + action='store_true', + help='Use a shaddow database to verify data integrity (default: false)') parser.add_argument( '-x', '--continue-on-exception', @@ -2414,6 +2463,11 @@ class MainExec: global gConfig gConfig = parser.parse_args() + crash_gen.settings.gConfig = gConfig # TODO: fix this hack, consolidate this global var + + # Sanity check for arguments + if gConfig.use_shadow_db and gConfig.max_dbs>1 : + raise CrashGenError("Cannot combine use-shadow-db with max-dbs of more than 1") Logging.clsInit(gConfig) diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index dc072d7abce68debd69cb162a4d784fcc5b68c4e..62a369c41a7ed0d73ab847232a206c2cabb53d53 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -15,8 +15,11 @@ from util.log import * from .misc import Logging, CrashGenError, Helper, Dice import os import datetime +import traceback # from .service_manager import TdeInstance +import crash_gen.settings + class DbConn: TYPE_NATIVE = "native-c" TYPE_REST = "rest-api" @@ -243,7 +246,7 @@ class MyTDSql: self._conn.close() # TODO: very important, cursor close does NOT close DB connection! self._cursor.close() - def _execInternal(self, sql): + def _execInternal(self, sql): startTime = time.time() # Logging.debug("Executing SQL: " + sql) ret = self._cursor.execute(sql) @@ -256,6 +259,27 @@ class MyTDSql: cls.longestQuery = sql cls.longestQueryTime = queryTime cls.lqStartTime = startTime + + # Now write to the shadow database + if crash_gen.settings.gConfig.use_shadow_db: + if sql[:11] == "INSERT INTO": + if sql[:16] == "INSERT INTO db_0": + sql2 = "INSERT INTO db_s" + sql[16:] + self._cursor.execute(sql2) + else: + raise CrashGenError("Did not find db_0 in INSERT statement: {}".format(sql)) + else: # not an insert statement + pass + + if sql[:12] == "CREATE TABLE": + if sql[:17] == "CREATE TABLE db_0": + sql2 = sql.replace('db_0', 'db_s') + self._cursor.execute(sql2) + else: + raise CrashGenError("Did not find db_0 in CREATE TABLE statement: {}".format(sql)) + else: # not an insert statement + pass + return ret def query(self, sql): @@ -301,12 +325,18 @@ class DbConnNative(DbConn): _lock = threading.Lock() # _connInfoDisplayed = False # TODO: find another way to display this totalConnections = 0 # Not private + totalRequests = 0 def __init__(self, dbTarget): super().__init__(dbTarget) self._type = self.TYPE_NATIVE self._conn = None - # self._cursor = None + # self._cursor = None + + @classmethod + def resetTotalRequests(cls): + with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!! + cls.totalRequests = 0 def openByType(self): # Open connection # global gContainer @@ -349,11 +379,14 @@ class DbConnNative(DbConn): def execute(self, sql): if (not self.isOpen): + traceback.print_stack() raise CrashGenError( "Cannot exec SQL unless db connection is open", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql nRows = self._tdSql.execute(sql) + cls = self.__class__ + cls.totalRequests += 1 Logging.debug( "[SQL] Execution Result, nRows = {}, SQL = {}".format( nRows, sql)) @@ -361,11 +394,14 @@ class DbConnNative(DbConn): def query(self, sql): # return rows affected if (not self.isOpen): + traceback.print_stack() raise CrashGenError( "Cannot query database until connection is open, restarting?", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql nRows = self._tdSql.query(sql) + cls = self.__class__ + cls.totalRequests += 1 Logging.debug( "[SQL] Query Result, nRows = {}, SQL = {}".format( nRows, sql)) diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index 6ea5691ce223eb1c14214d4b11c47cf85e29c795..9774ec5455961392d82ea2b4b59c0657b5704f9a 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -176,11 +176,13 @@ class Progress: SERVICE_START_NAP = 7 CREATE_TABLE_ATTEMPT = 8 QUERY_GROUP_BY = 9 + CONCURRENT_INSERTION = 10 + ACCEPTABLE_ERROR = 11 tokens = { STEP_BOUNDARY: '.', - BEGIN_THREAD_STEP: '[', - END_THREAD_STEP: '] ', + BEGIN_THREAD_STEP: ' [', + END_THREAD_STEP: ']', SERVICE_HEART_BEAT: '.Y.', SERVICE_RECONNECT_START: '', @@ -188,8 +190,14 @@ class Progress: SERVICE_START_NAP: '_zz', CREATE_TABLE_ATTEMPT: 'c', QUERY_GROUP_BY: 'g', + CONCURRENT_INSERTION: 'x', + ACCEPTABLE_ERROR: '_', } @classmethod def emit(cls, token): print(cls.tokens[token], end="", flush=True) + + @classmethod + def emitStr(cls, str): + print('({})'.format(str), end="", flush=True) diff --git a/tests/pytest/crash_gen/settings.py b/tests/pytest/crash_gen/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..3c4c91e6e077c325c53d15918624db783957fc20 --- /dev/null +++ b/tests/pytest/crash_gen/settings.py @@ -0,0 +1,8 @@ +from __future__ import annotations +import argparse + +gConfig: argparse.Namespace + +def init(): + global gConfig + gConfig = [] \ No newline at end of file diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index d9bc185b6c326b79d933dafc50cd7db0e0d396f4..bda52ffe6c7ab6cafe8a66fe18cdf4e3072759b9 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -1,5 +1,6 @@ #!/bin/bash ulimit -c unlimited +#======================p1-start=============== python3 ./test.py -f insert/basic.py python3 ./test.py -f insert/int.py @@ -19,7 +20,7 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py -#python3 ./test.py -f insert/before_1970.py +python3 ./test.py -f insert/before_1970.py python3 bug2265.py #table @@ -136,6 +137,38 @@ python3 ./test.py -f import_merge/importTPORestart.py python3 ./test.py -f import_merge/importTRestart.py python3 ./test.py -f import_merge/importInsertThenImport.py python3 ./test.py -f import_merge/importCSV.py +#======================p1-end=============== +#======================p2-start=============== +# tools +python3 test.py -f tools/taosdumpTest.py + +python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdemoTestWithoutMetric.py +python3 test.py -f tools/taosdemoTestWithJson.py +python3 test.py -f tools/taosdemoTestLimitOffset.py +python3 test.py -f tools/taosdemoTestTblAlt.py +python3 test.py -f tools/taosdemoTestSampleData.py +python3 test.py -f tools/taosdemoTestInterlace.py +python3 test.py -f tools/taosdemoTestQuery.py + +# update +python3 ./test.py -f update/allow_update.py +python3 ./test.py -f update/allow_update-0.py +python3 ./test.py -f update/append_commit_data.py +python3 ./test.py -f update/append_commit_last-0.py +python3 ./test.py -f update/append_commit_last.py +python3 ./test.py -f update/merge_commit_data.py + +python3 ./test.py -f update/merge_commit_data2.py +python3 ./test.py -f update/merge_commit_data2_update0.py +python3 ./test.py -f update/merge_commit_last-0.py +python3 ./test.py -f update/merge_commit_last.py +python3 ./test.py -f update/bug_td2279.py + +#======================p2-end=============== +#======================p3-start=============== + + # user python3 ./test.py -f user/user_create.py python3 ./test.py -f user/pass_len.py @@ -143,6 +176,11 @@ python3 ./test.py -f user/pass_len.py # stable python3 ./test.py -f stable/query_after_reset.py +# perfbenchmark +python3 ./test.py -f perfbenchmark/bug3433.py +python3 ./test.py -f perfbenchmark/bug3589.py + + #query python3 ./test.py -f query/filter.py python3 ./test.py -f query/filterCombo.py @@ -175,6 +213,11 @@ python3 ./test.py -f query/bug2119.py python3 ./test.py -f query/isNullTest.py python3 ./test.py -f query/queryWithTaosdKilled.py python3 ./test.py -f query/floatCompare.py +python3 ./test.py -f query/query1970YearsAf.py +python3 ./test.py -f query/bug3351.py +python3 ./test.py -f query/bug3375.py + + #stream python3 ./test.py -f stream/metric_1.py @@ -190,7 +233,6 @@ python3 ./test.py -f stream/table_n.py #alter table python3 ./test.py -f alter/alter_table_crash.py -python3 ./test.py -f alter/alter_table.py # client python3 ./test.py -f client/client.py @@ -203,6 +245,28 @@ python3 testCompress.py python3 testNoCompress.py python3 testMinTablesPerVnode.py + +python3 queryCount.py +python3 ./test.py -f query/queryGroupbyWithInterval.py +python3 client/twoClients.py +python3 test.py -f query/queryInterval.py +python3 test.py -f query/queryFillTest.py + +# subscribe +python3 test.py -f subscribe/singlemeter.py +#python3 test.py -f subscribe/stability.py +python3 test.py -f subscribe/supertable.py + + +#======================p3-end=============== +#======================p4-start=============== + +python3 ./test.py -f update/merge_commit_data-0.py +# wal +python3 ./test.py -f wal/addOldWalTest.py + +# function +python3 ./test.py -f functions/all_null_value.py # functions python3 ./test.py -f functions/function_avg.py -r 1 python3 ./test.py -f functions/function_bottom.py -r 1 @@ -222,40 +286,24 @@ python3 ./test.py -f functions/function_sum.py -r 1 python3 ./test.py -f functions/function_top.py -r 1 python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py -python3 ./test.py -f functions/all_null_value.py -python3 queryCount.py -python3 ./test.py -f query/queryGroupbyWithInterval.py -python3 client/twoClients.py -python3 ./test.py -f query/queryInterval.py -python3 ./test.py -f query/queryFillTest.py -python3 ./test.py -f query/last_row_cache.py +python3 ./test.py -f functions/function_stddev_td2555.py +python3 ./test.py -f insert/metadataUpdate.py python3 ./test.py -f query/last_cache.py +python3 ./test.py -f query/last_row_cache.py +python3 ./test.py -f account/account_create.py +python3 ./test.py -f alter/alter_table.py +python3 ./test.py -f query/queryGroupbySort.py -# tools -python3 test.py -f tools/taosdemoTest.py -python3 test.py -f tools/taosdumpTest.py -python3 test.py -f tools/lowaTest.py -python3 test.py -f tools/taosdemoTest2.py - -# subscribe -python3 test.py -f subscribe/singlemeter.py -#python3 test.py -f subscribe/stability.py -python3 test.py -f subscribe/supertable.py - +python3 ./test.py -f insert/unsignedInt.py +python3 ./test.py -f insert/unsignedBigint.py +python3 ./test.py -f insert/unsignedSmallint.py +python3 ./test.py -f insert/unsignedTinyint.py +python3 ./test.py -f query/filterAllUnsignedIntTypes.py -# update -python3 ./test.py -f update/allow_update.py -python3 ./test.py -f update/allow_update-0.py -python3 ./test.py -f update/append_commit_data.py -python3 ./test.py -f update/append_commit_last-0.py -python3 ./test.py -f update/append_commit_last.py -python3 ./test.py -f update/merge_commit_data.py -python3 ./test.py -f update/merge_commit_data-0.py -python3 ./test.py -f update/merge_commit_data2.py -python3 ./test.py -f update/merge_commit_data2_update0.py -python3 ./test.py -f update/merge_commit_last-0.py -python3 ./test.py -f update/merge_commit_last.py -python3 ./test.py -f update/bug_td2279.py +python3 ./test.py -f functions/function_percentile2.py +python3 ./test.py -f insert/boundary2.py +python3 ./test.py -f alter/alter_debugFlag.py +python3 ./test.py -f query/queryBetweenAnd.py +python3 ./test.py -f tag_lite/alter_tag.py -# wal -python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file +#======================p4-end=============== diff --git a/tests/pytest/functions/function_avg.py b/tests/pytest/functions/function_avg.py index 9481550ba3f9d912b734d19b7c794c495fffdecd..c8a1cca77b1568b760b0208e0fa71583ae2e3645 100644 --- a/tests/pytest/functions/function_avg.py +++ b/tests/pytest/functions/function_avg.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -63,7 +63,15 @@ class TDTestCase: tdSql.query("select avg(col5) from test") tdSql.checkData(0, 0, np.average(floatData)) tdSql.query("select avg(col6) from test") - tdSql.checkData(0, 0, np.average(floatData)) + tdSql.checkData(0, 0, np.average(floatData)) + tdSql.query("select avg(col11) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col12) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col13) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col14) from test") + tdSql.checkData(0, 0, np.average(intData)) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_avg_restart.py b/tests/pytest/functions/function_avg_restart.py index 56b99cdf910869b5bc4f5bd93da208943b1b026e..60f3ba7d4e2708053425b4866e072516b5ff961c 100644 --- a/tests/pytest/functions/function_avg_restart.py +++ b/tests/pytest/functions/function_avg_restart.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - #tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + #tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -63,7 +63,15 @@ class TDTestCase: tdSql.query("select avg(col5) from test") tdSql.checkData(0, 0, np.average(floatData)) tdSql.query("select avg(col6) from test") - tdSql.checkData(0, 0, np.average(floatData)) + tdSql.checkData(0, 0, np.average(floatData)) + tdSql.query("select avg(col11) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col12) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col13) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col14) from test") + tdSql.checkData(0, 0, np.average(intData)) def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py index 3cc389221845b38d92ca130e086d50fddc344d6d..abb9ac48e70ac876741c3da2a9f223d537c8644b 100644 --- a/tests/pytest/functions/function_bottom.py +++ b/tests/pytest/functions/function_bottom.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # bottom verifacation tdSql.error("select bottom(ts, 10) from test") @@ -84,6 +84,26 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(0, 1, 0.1) tdSql.checkData(1, 1, 1.1) + + tdSql.query("select bottom(col11, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col12, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col13, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col14, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) #TD-2457 bottom + interval + order by tdSql.error('select top(col2,1) from test interval(1y) order by col2;') diff --git a/tests/pytest/functions/function_bottom_restart.py b/tests/pytest/functions/function_bottom_restart.py index 74eb044645ffa6a54e286dd11d022672575a38db..b4d987975c3660e2ea875a03f3406eaebcafea6d 100644 --- a/tests/pytest/functions/function_bottom_restart.py +++ b/tests/pytest/functions/function_bottom_restart.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.execute("use db") #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") #for i in range(self.rowNum): - # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # bottom verifacation tdSql.error("select bottom(ts, 10) from test") @@ -75,6 +75,26 @@ class TDTestCase: tdSql.checkData(0, 1, 1) tdSql.checkData(1, 1, 2) + tdSql.query("select bottom(col11, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col12, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col13, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col14, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + tdSql.query("select bottom(col5, 2) from test") tdSql.checkRows(2) tdSql.checkData(0, 1, 0.1) diff --git a/tests/pytest/functions/function_count.py b/tests/pytest/functions/function_count.py index 4795a6b2dfe72bb9fa747b550be9f56c0c255d88..9812473d6496a94447734f1f3507ac806e392294 100644 --- a/tests/pytest/functions/function_count.py +++ b/tests/pytest/functions/function_count.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # Count verifacation tdSql.query("select count(*) from test") @@ -62,11 +62,20 @@ class TDTestCase: tdSql.query("select count(col9) from test") tdSql.checkData(0, 0, 10) + tdSql.query("select count(col11) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col12) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col13) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col14) from test") + tdSql.checkData(0, 0, 10) + tdSql.execute("alter table test add column col10 int") tdSql.query("select count(col10) from test") tdSql.checkRows(0) - tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' 1)") + tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' , 1, 1, 1, 1, 1)") tdSql.query("select count(col10) from test") tdSql.checkData(0, 0, 1) diff --git a/tests/pytest/functions/function_count_restart.py b/tests/pytest/functions/function_count_restart.py index 5eabb47d952a1a461b6fd5c976f3f2a8c294222b..565d85f4c0712b436e2dc24335aa13f60d3c87dd 100644 --- a/tests/pytest/functions/function_count_restart.py +++ b/tests/pytest/functions/function_count_restart.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.execute("use db") #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") #for i in range(self.rowNum): - # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # Count verifacation tdSql.query("select count(*) from test") @@ -62,6 +62,15 @@ class TDTestCase: tdSql.query("select count(col9) from test") tdSql.checkData(0, 0, 11) + tdSql.query("select count(col11) from test") + tdSql.checkData(0, 0, 11) + tdSql.query("select count(col12) from test") + tdSql.checkData(0, 0, 11) + tdSql.query("select count(col13) from test") + tdSql.checkData(0, 0, 11) + tdSql.query("select count(col14) from test") + tdSql.checkData(0, 0, 11) + #tdSql.execute("alter table test add column col10 int") #tdSql.query("select count(col10) from test") #tdSql.checkRows(0) diff --git a/tests/pytest/functions/function_diff.py b/tests/pytest/functions/function_diff.py index b6f496d7d68ebaa0f19dfc947d7745802198a3dc..46ea3a86deee75d26f0b635ae0d3b2c5673fb6da 100644 --- a/tests/pytest/functions/function_diff.py +++ b/tests/pytest/functions/function_diff.py @@ -31,9 +31,9 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") - tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ', 0, 0, 0, 0)" % (self.ts - 1)) # diff verifacation tdSql.query("select diff(col1) from test1") @@ -55,8 +55,8 @@ class TDTestCase: tdSql.checkRows(0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.error("select diff(ts) from test") tdSql.error("select diff(ts) from test1") @@ -72,7 +72,16 @@ class TDTestCase: tdSql.error("select diff(col8) from test1") tdSql.error("select diff(col9) from test") tdSql.error("select diff(col9) from test1") - + tdSql.error("select diff(col11) from test1") + tdSql.error("select diff(col12) from test1") + tdSql.error("select diff(col13) from test1") + tdSql.error("select diff(col14) from test1") + tdSql.error("select diff(col11) from test") + tdSql.error("select diff(col12) from test") + tdSql.error("select diff(col13) from test") + tdSql.error("select diff(col14) from test") + + tdSql.query("select diff(col1) from test1") tdSql.checkRows(10) diff --git a/tests/pytest/functions/function_diff_restart.py b/tests/pytest/functions/function_diff_restart.py index 870ee03fc94cbd8dbf5e858726cd52b09010a884..e03b892477eba46c007163049d91012af236e730 100644 --- a/tests/pytest/functions/function_diff_restart.py +++ b/tests/pytest/functions/function_diff_restart.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.execute("use db") #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") #tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) @@ -55,8 +55,8 @@ class TDTestCase: #tdSql.checkRows(0) #for i in range(self.rowNum): - # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + # tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.error("select diff(ts) from test") tdSql.error("select diff(ts) from test1") @@ -71,7 +71,14 @@ class TDTestCase: tdSql.error("select diff(col8) from test") tdSql.error("select diff(col8) from test1") tdSql.error("select diff(col9) from test") - tdSql.error("select diff(col9) from test1") + tdSql.error("select diff(col11) from test1") + tdSql.error("select diff(col12) from test1") + tdSql.error("select diff(col13) from test1") + tdSql.error("select diff(col14) from test1") + tdSql.error("select diff(col11) from test") + tdSql.error("select diff(col12) from test") + tdSql.error("select diff(col13) from test") + tdSql.error("select diff(col14) from test") tdSql.query("select diff(col1) from test1") tdSql.checkRows(10) diff --git a/tests/pytest/functions/function_first.py b/tests/pytest/functions/function_first.py index 5b2aacb779a64377baa0417d5ba0ba5ef0a27290..f1a916b168271e67d0001c5d4444966f8c07a2d1 100644 --- a/tests/pytest/functions/function_first.py +++ b/tests/pytest/functions/function_first.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) @@ -52,6 +52,18 @@ class TDTestCase: tdSql.query("select first(col4) from test1") tdSql.checkRows(0) + tdSql.query("select first(col11) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col12) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col13) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col14) from test1") + tdSql.checkRows(0) + tdSql.query("select first(col5) from test1") tdSql.checkRows(0) @@ -68,8 +80,8 @@ class TDTestCase: tdSql.checkRows(0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.query("select first(*) from test1") tdSql.checkRows(1) @@ -91,6 +103,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 1) + tdSql.query("select first(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + tdSql.query("select first(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 0.1) diff --git a/tests/pytest/functions/function_last.py b/tests/pytest/functions/function_last.py index 337c18a28c2cd99d556e5b017377edc56671c490..991ac96a800803440a2e662c163622af95c556e3 100644 --- a/tests/pytest/functions/function_last.py +++ b/tests/pytest/functions/function_last.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) @@ -52,6 +52,18 @@ class TDTestCase: tdSql.query("select last(col4) from test1") tdSql.checkRows(0) + tdSql.query("select last(col11) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col12) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col13) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col14) from test1") + tdSql.checkRows(0) + tdSql.query("select last(col5) from test1") tdSql.checkRows(0) @@ -68,8 +80,8 @@ class TDTestCase: tdSql.checkRows(0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.query("select last(*) from test1") tdSql.checkRows(1) @@ -91,6 +103,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 10) + tdSql.query("select last(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) diff --git a/tests/pytest/functions/function_last_row.py b/tests/pytest/functions/function_last_row.py index ea5cf661eb9e70e2ed06ebd712b2f65dfa62beeb..d075eeeca55c7e6575ab9bd526ddb6177bf61791 100644 --- a/tests/pytest/functions/function_last_row.py +++ b/tests/pytest/functions/function_last_row.py @@ -31,7 +31,7 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) @@ -56,6 +56,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, None) + tdSql.query("select last_row(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + tdSql.query("select last_row(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, None) @@ -77,8 +93,8 @@ class TDTestCase: tdSql.checkData(0, 0, None) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.query("select last_row(*) from test1") tdSql.checkRows(1) @@ -100,6 +116,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 10) + tdSql.query("select last_row(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select last_row(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) diff --git a/tests/pytest/functions/function_leastsquares.py b/tests/pytest/functions/function_leastsquares.py index 0ef0a2b78612dca604f7a7a7f70841c917376a17..3b2a8486d5e6d8bbc2464b8ee391181e10e6a285 100644 --- a/tests/pytest/functions/function_leastsquares.py +++ b/tests/pytest/functions/function_leastsquares.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # leastsquares verifacation tdSql.error("select leastsquares(ts, 1, 1) from test1") @@ -48,6 +48,10 @@ class TDTestCase: tdSql.error("select leastsquares(col7, 1, 1) from test1") tdSql.error("select leastsquares(col8, 1, 1) from test1") tdSql.error("select leastsquares(col9, 1, 1) from test1") + tdSql.error("select leastsquares(col11, 1, 1) from test") + tdSql.error("select leastsquares(col12, 1, 1) from test") + tdSql.error("select leastsquares(col13, 1, 1) from test") + tdSql.error("select leastsquares(col14, 1, 1) from test") tdSql.query("select leastsquares(col1, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') @@ -61,6 +65,18 @@ class TDTestCase: tdSql.query("select leastsquares(col4, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + tdSql.query("select leastsquares(col11, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col12, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col13, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col14, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + tdSql.query("select leastsquares(col5, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') diff --git a/tests/pytest/functions/function_leastsquares_restart.py b/tests/pytest/functions/function_leastsquares_restart.py index 0221dc0aed865882e375f90c44f52b391d1ffee5..8c38d5dd95ccd4537af623253b08fb2253e4d140 100644 --- a/tests/pytest/functions/function_leastsquares_restart.py +++ b/tests/pytest/functions/function_leastsquares_restart.py @@ -54,6 +54,18 @@ class TDTestCase: tdSql.query("select leastsquares(col4, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + tdSql.query("select leastsquares(col11, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col12, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col13, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col14, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + tdSql.query("select leastsquares(col5, 1, 1) from test1") tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') diff --git a/tests/pytest/functions/function_max.py b/tests/pytest/functions/function_max.py index d1f8f75892ce2e61388c107eb7cad1ed00582d2f..c322b6af26362949f7e8b45a58e4c9cef3d9de19 100644 --- a/tests/pytest/functions/function_max.py +++ b/tests/pytest/functions/function_max.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -63,6 +63,18 @@ class TDTestCase: tdSql.query("select max(col4) from test1") tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col11) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col12) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col13) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col14) from test1") + tdSql.checkData(0, 0, np.max(intData)) tdSql.query("select max(col5) from test1") tdSql.checkData(0, 0, np.max(floatData)) diff --git a/tests/pytest/functions/function_max_restart.py b/tests/pytest/functions/function_max_restart.py index 8d1827561758f1ceebffab877aabd6cd79166d42..a04fcedd8b908b030932d5d182b8cf20c28e2685 100644 --- a/tests/pytest/functions/function_max_restart.py +++ b/tests/pytest/functions/function_max_restart.py @@ -34,7 +34,7 @@ class TDTestCase: floatData = [] #tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - # col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + # col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') #tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): intData.append(i + 1) @@ -61,6 +61,18 @@ class TDTestCase: tdSql.query("select max(col4) from test1") tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col11) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col12) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col13) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col14) from test1") + tdSql.checkData(0, 0, np.max(intData)) tdSql.query("select max(col5) from test1") tdSql.checkData(0, 0, np.max(floatData)) diff --git a/tests/pytest/functions/function_min.py b/tests/pytest/functions/function_min.py index c779744ced63d83fd5b2116fb61dfa347ec4dd5a..b4d6d58f7ce01b97dd7f5f569409a0288fb0389b 100644 --- a/tests/pytest/functions/function_min.py +++ b/tests/pytest/functions/function_min.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -63,6 +63,18 @@ class TDTestCase: tdSql.query("select min(col4) from test1") tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col11) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col12) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col13) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col14) from test1") + tdSql.checkData(0, 0, np.min(intData)) tdSql.query("select min(col5) from test1") tdSql.checkData(0, 0, np.min(floatData)) diff --git a/tests/pytest/functions/function_min_restart.py b/tests/pytest/functions/function_min_restart.py index c8329bffa5525c2386d264010acf7231059e344d..fc2cb761cae164c8c04992768133d1e00a16fccb 100644 --- a/tests/pytest/functions/function_min_restart.py +++ b/tests/pytest/functions/function_min_restart.py @@ -58,6 +58,18 @@ class TDTestCase: tdSql.query("select min(col4) from test1") tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col11) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col12) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col13) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col14) from test1") + tdSql.checkData(0, 0, np.min(intData)) tdSql.query("select min(col5) from test1") tdSql.checkData(0, 0, np.min(floatData)) diff --git a/tests/pytest/functions/function_operations.py b/tests/pytest/functions/function_operations.py index 930c23d8a60d84bcc57fa8952e4d8c8e847ec138..162aa3eb658a60615c9f20de0c363f9066096f0c 100644 --- a/tests/pytest/functions/function_operations.py +++ b/tests/pytest/functions/function_operations.py @@ -31,11 +31,11 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) # min verifacation tdSql.error("select ts + col1 from test") @@ -51,9 +51,9 @@ class TDTestCase: tdSql.checkRows(10) tdSql.checkData(0, 0, 2.0) - tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 + col11 + col12 + col13 + col14 from test1") tdSql.checkRows(10) - tdSql.checkData(0, 0, 3.2) + tdSql.checkData(0, 0, 7.2) tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11)) tdSql.query("select col1 + col2 from test1") @@ -64,11 +64,44 @@ class TDTestCase: tdSql.checkRows(11) tdSql.checkData(10, 0, None) - tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 + col11 + col12 + col13 + col14 from test1") tdSql.checkRows(11) tdSql.checkData(10, 0, None) + # test for tarithoperator.c coverage + tdSql.execute("insert into test1 values(1537146000010,1,NULL,9,8,1.2,1.3,0,1,1,5,4,3,2)") + tdSql.execute("insert into test1 values(1537146000011,2,1,NULL,9,1.2,1.3,1,2,2,6,5,4,3)") + tdSql.execute("insert into test1 values(1537146000012,3,2,1,NULL,1.2,1.3,0,3,3,7,6,5,4)") + tdSql.execute("insert into test1 values(1537146000013,4,3,2,1,1.2,1.3,1,4,4,8,7,6,5)") + tdSql.execute("insert into test1 values(1537146000014,5,4,3,2,1.2,1.3,0,5,5,9,8,7,6)") + tdSql.execute("insert into test1 values(1537146000015,6,5,4,3,1.2,1.3,1,6,6,NULL,9,8,7)") + tdSql.execute("insert into test1 values(1537146000016,7,6,5,4,1.2,1.3,0,7,7,1,NULL,9,8)") + tdSql.execute("insert into test1 values(1537146000017,8,7,6,5,1.2,1.3,1,8,8,2,1,NULL,9)") + tdSql.execute("insert into test1 values(1537146000018,9,8,7,6,1.2,1.3,0,9,9,3,2,1,NULL)") + tdSql.execute("insert into test1 values(1537146000019,NULL,9,8,7,1.2,1.3,1,10,10,4,3,2,1)") + self.ts = self.ts + self.rowNum + 10 + + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, 1.1, 1, NULL, '涛思数据3', 1, 1, 1, 1)" % ( self.ts + self.rowNum + 1 )) + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, 1.1, 1, 'taosdata', NULL, 1, 1, 1, 1)" % ( self.ts + self.rowNum + 2 )) + tdSql.execute("insert into test1 values(%d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" % ( self.ts + self.rowNum + 3 )) + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, NULL, 1.1, 1, NULL, '涛思数据3', 1, 1, 1, 1)" % ( self.ts + self.rowNum + 4 )) + tdSql.execute("insert into test1 values(%d, 1, 1, 1, 1, 1.1, NULL, 1, 'taosdata', NULL, 1, 1, 1, 1)" % ( self.ts + self.rowNum + 5 )) + self.rowNum = self.rowNum + 5 + + col_list = [ 'col1' , 'col2' , 'col3' , 'col4' , 'col5' , 'col6' , 'col7' , 'col8' , 'col9' , 'col11' , 'col12' , 'col13' , 'col14' , '1' , '1.1' , 'NULL' ] + op_list = [ '+' , '-' , '*' , '/' , '%' ] + err_list = [ 'col7' , 'col8' , 'col9' , 'NULL' ] + order_lsit = [ ' order by ts ', ' order by ts desc ', ' order by ts asc '] + for i in col_list : + for j in col_list : + for k in op_list : + for l in order_lsit : + sql = " select %s %s %s from test1 %s" % ( i , k , j , l ) + if i in err_list or j in err_list: + tdSql.error(sql) + else: + tdSql.query(sql) def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/functions/function_operations_restart.py b/tests/pytest/functions/function_operations_restart.py index 6e3990264cb91d8c063fe638a5f77231ef7be424..3ba787a4a0f3f8af74f461f4e668078d726a428d 100644 --- a/tests/pytest/functions/function_operations_restart.py +++ b/tests/pytest/functions/function_operations_restart.py @@ -41,24 +41,24 @@ class TDTestCase: tdSql.error("select col1 + col9 from test1") tdSql.query("select col1 + col2 from test1") - tdSql.checkRows(11) + tdSql.checkRows(25) tdSql.checkData(0, 0, 2.0) - tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") - tdSql.checkRows(11) - tdSql.checkData(0, 0, 3.2) + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 + col11 + col12 + col13 + col14 from test1") + tdSql.checkRows(25) + tdSql.checkData(0, 0, 7.2) #tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11)) tdSql.query("select col1 + col2 from test1") - tdSql.checkRows(11) + tdSql.checkRows(25) tdSql.checkData(10, 0, None) tdSql.query("select col1 + col2 * col3 from test1") - tdSql.checkRows(11) + tdSql.checkRows(25) tdSql.checkData(10, 0, None) - tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") - tdSql.checkRows(11) + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 + col11 + col12 + col13 + col14 from test1") + tdSql.checkRows(25) tdSql.checkData(10, 0, None) diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py index 688e91eefa79348a88f37f3b7cb0cc1ac63152d8..9b5e209d33d1f8e97c5baf83898dd62746c11dd8 100644 --- a/tests/pytest/functions/function_percentile.py +++ b/tests/pytest/functions/function_percentile.py @@ -34,10 +34,10 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') for i in range(self.rowNum): - tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -103,6 +103,58 @@ class TDTestCase: tdSql.query("select apercentile(col4, 100) from test") print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col11, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col11, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col11, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col12, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col12, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col12, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col12, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col12, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col12, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col13, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col13, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col13, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col13, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col13, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col13, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col14, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col14, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col14, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col14, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col14, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col14, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 0) from test") print("query result: %s" % tdSql.getData(0, 0)) print("array result: %s" % np.percentile(floatData, 0)) diff --git a/tests/pytest/functions/function_percentile2.py b/tests/pytest/functions/function_percentile2.py new file mode 100644 index 0000000000000000000000000000000000000000..227a3cbe0307a8a198fed2e981eba2890f1e5664 --- /dev/null +++ b/tests/pytest/functions/function_percentile2.py @@ -0,0 +1,63 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 1000000 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table test(ts timestamp, col1 int, col2 float, col3 double)") + for i in range(1000): + sql = "insert into test values" + batchSize = int (self.rowNum / 1000) + for j in range (batchSize): + currTime = self.ts + batchSize * i + j + sql += "(%d, 1, 2.37, 3.1415926)" % currTime + tdSql.execute(sql) + + tdSql.query("select percentile(col1, 20) from test") + tdSql.checkData(0, 0, 1) + + tdSql.query("select percentile(col2, 20) from test") + tdSql.checkData(0, 0, 2.3699998) + + tdSql.query("select percentile(col3, 20) from test") + tdSql.checkData(0, 0, 3.1415926) + + tdSql.query("select apercentile(col1, 20) from test") + tdSql.checkData(0, 0, 1) + + tdSql.query("select apercentile(col2, 20) from test") + tdSql.checkData(0, 0, 2.3699998) + + tdSql.query("select apercentile(col3, 20) from test") + tdSql.checkData(0, 0, 3.1415926) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_percentile_restart.py b/tests/pytest/functions/function_percentile_restart.py index ade10282dedd655bc7ccb912a30a2a3e88a72223..5e6898784b55bbffb5e01946fff9dc60e281ebff 100644 --- a/tests/pytest/functions/function_percentile_restart.py +++ b/tests/pytest/functions/function_percentile_restart.py @@ -99,6 +99,58 @@ class TDTestCase: tdSql.query("select apercentile(col4, 100) from test") print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col11, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col11, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col11, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col11, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col12, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col12, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col12, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col12, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col12, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col12, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col13, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col13, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col13, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col13, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col13, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col13, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col14, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col14, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col14, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col14, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col14, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col14, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 0) from test") print("query result: %s" % tdSql.getData(0, 0)) print("array result: %s" % np.percentile(floatData, 0)) diff --git a/tests/pytest/functions/function_spread.py b/tests/pytest/functions/function_spread.py index 462933597388090f069f9319751aaea032ea4133..b40f57b78da0f937e06bd304764c6b660f82488e 100644 --- a/tests/pytest/functions/function_spread.py +++ b/tests/pytest/functions/function_spread.py @@ -31,9 +31,9 @@ class TDTestCase: tdSql.prepare() tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") - tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ', 0, 0, 0, 0)" % (self.ts - 1)) # spread verifacation tdSql.query("select spread(ts) from test1") @@ -55,6 +55,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 0) + tdSql.query("select spread(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + tdSql.query("select spread(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 0) @@ -64,8 +80,8 @@ class TDTestCase: tdSql.checkData(0, 0, 0) for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) tdSql.error("select spread(col7) from test") tdSql.error("select spread(col7) from test1") @@ -90,6 +106,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 10) + tdSql.query("select spread(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + tdSql.query("select spread(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) diff --git a/tests/pytest/functions/function_spread_restart.py b/tests/pytest/functions/function_spread_restart.py index ba71da0a18ac2e1e2a1ead6ab3de9ff5cae6a600..134d92c9c38c1ae37e9ea1a0a91d0e93957a7e2a 100644 --- a/tests/pytest/functions/function_spread_restart.py +++ b/tests/pytest/functions/function_spread_restart.py @@ -36,7 +36,7 @@ class TDTestCase: tdSql.error("select spread(col8) from test1") tdSql.error("select spread(col9) from test") tdSql.error("select spread(col9) from test1") - + tdSql.query("select spread(col1) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 10) @@ -53,6 +53,23 @@ class TDTestCase: tdSql.checkRows(1) tdSql.checkData(0, 0, 10) + tdSql.query("select spread(col11) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col12) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col13) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col14) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col5) from test1") tdSql.checkRows(1) tdSql.checkData(0, 0, 9.1) diff --git a/tests/pytest/functions/function_stddev.py b/tests/pytest/functions/function_stddev.py index 23df415aa35feecf84ba40a50604d2b86fc7f008..3ff2b82bf6b326ed4d07a5a51027c9e266c2fd72 100644 --- a/tests/pytest/functions/function_stddev.py +++ b/tests/pytest/functions/function_stddev.py @@ -26,6 +26,8 @@ class TDTestCase: self.rowNum = 10 self.ts = 1537146000000 + self.stb_prefix = 's' + self.subtb_prefix = 't' def run(self): tdSql.prepare() @@ -34,22 +36,24 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) # stddev verifacation tdSql.error("select stddev(ts) from test1") - tdSql.error("select stddev(col1) from test") - tdSql.error("select stddev(col2) from test") - tdSql.error("select stddev(col3) from test") - tdSql.error("select stddev(col4) from test") - tdSql.error("select stddev(col5) from test") - tdSql.error("select stddev(col6) from test") + + # stddev support super table now + # tdSql.error("select stddev(col1) from test") + # tdSql.error("select stddev(col2) from test") + # tdSql.error("select stddev(col3) from test") + # tdSql.error("select stddev(col4) from test") + # tdSql.error("select stddev(col5) from test") + # tdSql.error("select stddev(col6) from test") tdSql.error("select stddev(col7) from test1") tdSql.error("select stddev(col8) from test1") tdSql.error("select stddev(col9) from test1") @@ -66,11 +70,60 @@ class TDTestCase: tdSql.query("select stddev(col4) from test1") tdSql.checkData(0, 0, np.std(intData)) + tdSql.query("select stddev(col11) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col12) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col13) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col14) from test1") + tdSql.checkData(0, 0, np.std(intData)) + tdSql.query("select stddev(col5) from test1") tdSql.checkData(0, 0, np.std(floatData)) tdSql.query("select stddev(col6) from test1") tdSql.checkData(0, 0, np.std(floatData)) + + #add for td-3276 + sql="create table s (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20),c11 int unsigned,c12 smallint unsigned,c13 tinyint unsigned,c14 bigint unsigned) \ + tags(t1 int, t2 float, t3 bigint, t4 smallint, t5 tinyint, t6 double, t7 bool,t8 binary(20),t9 nchar(20), t10 int unsigned , t11 smallint unsigned , t12 tinyint unsigned , t13 bigint unsigned)" + tdSql.execute(sql) + for j in range(2): + if j % 2 == 0: + sql = "create table %s using %s tags(NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % \ + (self.subtb_prefix+str(j)+'_'+str(j),self.stb_prefix) + else: + sql = "create table %s using %s tags(%d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % \ + (self.subtb_prefix+str(j)+'_'+str(j),self.stb_prefix,j,j/2.0,j%41,j%51,j%53,j*1.0,j%2,'taos'+str(j),'涛思'+str(j), j%43, j%23 , j%17 , j%3167) + tdSql.execute(sql) + for i in range(10): + if i % 5 == 0 : + ret = tdSql.execute( + "insert into %s values (%d , NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)" % + (self.subtb_prefix+str(j)+'_'+str(j), self.ts+i)) + else: + ret = tdSql.execute( + "insert into %s values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s',%d,%d,%d,%d)" % + (self.subtb_prefix+str(j)+'_'+str(j), self.ts+i, i%100, i/2.0, i%41, i%51, i%53, i*1.0, i%2,'taos'+str(i),'涛思'+str(i), i%43, i%23 , i%17 , i%3167)) + + for i in range(13): + tdSql.query('select stddev(c4) from s group by t%s' % str(i+1) ) + + #add for td-3223 + for i in range(13): + if i == 1 or i == 5 or i == 6 or i == 7 or i == 9 or i == 8 :continue + tdSql.query('select stddev(c%d),stddev(c%d) from s group by c%d' %( i+1 , i+1 , i+1 ) ) + + #add for TD-3318 + tdSql.execute('create table t1(ts timestamp, k int, b binary(12));') + tdSql.execute("insert into t1 values(now, 1, 'abc');") + tdLog.info("select stddev(k) from t1 where b <> 'abc' interval(1s);") + tdSql.query("select stddev(k) from t1 where b <> 'abc' interval(1s);") + def stop(self): tdSql.close() diff --git a/tests/pytest/functions/function_stddev_restart.py b/tests/pytest/functions/function_stddev_restart.py index ec413b94b20a89c210131188d2a09e38cdbc721c..48f9821d89473b66134b5ece06f1d0c417c24b36 100644 --- a/tests/pytest/functions/function_stddev_restart.py +++ b/tests/pytest/functions/function_stddev_restart.py @@ -39,12 +39,6 @@ class TDTestCase: # stddev verifacation tdSql.error("select stddev(ts) from test1") - tdSql.error("select stddev(col1) from test") - tdSql.error("select stddev(col2) from test") - tdSql.error("select stddev(col3) from test") - tdSql.error("select stddev(col4) from test") - tdSql.error("select stddev(col5) from test") - tdSql.error("select stddev(col6) from test") tdSql.error("select stddev(col7) from test1") tdSql.error("select stddev(col8) from test1") tdSql.error("select stddev(col9) from test1") @@ -61,6 +55,18 @@ class TDTestCase: tdSql.query("select stddev(col4) from test1") tdSql.checkData(0, 0, np.std(intData)) + tdSql.query("select stddev(col11) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col12) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col13) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col14) from test1") + tdSql.checkData(0, 0, np.std(intData)) + tdSql.query("select stddev(col5) from test1") tdSql.checkData(0, 0, np.std(floatData)) diff --git a/tests/pytest/functions/function_stddev_td2555.py b/tests/pytest/functions/function_stddev_td2555.py new file mode 100644 index 0000000000000000000000000000000000000000..d7c820c37644397d18cc090afb1ee82efeaecf6c --- /dev/null +++ b/tests/pytest/functions/function_stddev_td2555.py @@ -0,0 +1,105 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 100 + self.ts = 1537146000000 + self.clist1 = [] + self.clist2 = [] + self.clist3 = [] + self.clist4 = [] + self.clist5 = [] + self.clist6 = [] + self.clist11 = [] + self.clist12 = [] + self.clist13 = [] + self.clist14 = [] + + def getData(self): + for i in range(tdSql.queryRows): + for j in range(6): + exec('self.clist{}.append(tdSql.queryResult[i][j+1])'.format(j+1)) + for j in range(11,15): + exec('self.clist{}.append(tdSql.queryResult[i][j-1])'.format(j)) + + def run(self): + tdSql.prepare() + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(cid int,gbid binary(20),loc nchar(20))''') + tdSql.execute("create table test1 using test tags(1,'beijing','北京')") + tdSql.execute("create table test2 using test tags(2,'shanghai','深圳')") + tdSql.execute("create table test3 using test tags(2,'shenzhen','深圳')") + tdSql.execute("create table test4 using test tags(1,'shanghai','上海')") + for j in range(4): + for i in range(self.rowNum): + tdSql.execute("insert into test%d values(now-%dh, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (j+1,i, i + 1, i + 1, i + 1, i + 1, i + i * 0.1, i * 1.5, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + # stddev verifacation + tdSql.error("select stddev(ts) from test") + tdSql.error("select stddev(col7) from test") + tdSql.error("select stddev(col8) from test") + tdSql.error("select stddev(col9) from test") + + con_list = [ + ' where cid = 1 and ts >=now - 1d and ts =now - 1d and ts =now - 1d and ts =now - 1d and ts =now - 1d and ts %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col11) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col11) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col12) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col12) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col13) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col13) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col14) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col14) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) diff --git a/tests/pytest/functions/function_twa_restart.py b/tests/pytest/functions/function_twa_restart.py index 56242c29535646f1baf12948ad06ebbe1a7f08ac..a066a5a8f3a4e7f28c577fec0ace725c920d5911 100644 --- a/tests/pytest/functions/function_twa_restart.py +++ b/tests/pytest/functions/function_twa_restart.py @@ -34,11 +34,11 @@ class TDTestCase: floatData = [] tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, - col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(loc nchar(20))''') tdSql.execute("create table test1 using test tags('beijing')") for i in range(self.rowNum): - tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) intData.append(i + 1) floatData.append(i + 0.1) @@ -52,7 +52,15 @@ class TDTestCase: tdSql.error("select twa(col3) from test") - tdSql.error("select twa(col4) from test") + tdSql.error("select twa(col4) from test") + + tdSql.error("select twa(col11) from test") + + tdSql.error("select twa(col12) from test") + + tdSql.error("select twa(col13) from test") + + tdSql.error("select twa(col14) from test") tdSql.error("select twa(col5) from test") @@ -79,6 +87,18 @@ class TDTestCase: tdSql.error("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col11) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col11) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col12) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col12) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col13) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col13) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.error("select twa(col14) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col14) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index 528316700d184171641f7f686a3c12102b6c1feb..55c10639d7af941eda21cf33cbf7050c95fe3539 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -16,7 +16,7 @@ TOP_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep -v community|head -n1` nohup $TAOSD_DIR >/dev/null & cd - -./crash_gen.sh --valgrind -p -t 10 -s 350 -b 4 +./crash_gen.sh --valgrind -p -t 10 -s 1000 -b 4 pidof taosd|xargs kill -9 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log @@ -36,11 +36,13 @@ for defiMemError in `grep 'definitely lost:' crash_gen-definitely-lost-out.log | do defiMemError=(${defiMemError//,/}) if [ -n "$defiMemError" ]; then - if [ "$defiMemError" -gt 3 -a "$defiMemError" -lt 1013 ]; then - echo -e "${RED} ## Memory errors number valgrind reports \ - Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + if [ "$defiMemError" -gt 0 -a "$defiMemError" -lt 1013 ]; then + cat valgrind.err + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" exit 8 elif [ "$defiMemError" -gt 1013 ];then #add for azure + cat valgrind.err echo -e "${RED} ## Memory errors number valgrind reports \ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" exit 8 diff --git a/tests/pytest/handle_taosd_val_log.sh b/tests/pytest/handle_taosd_val_log.sh new file mode 100755 index 0000000000000000000000000000000000000000..6b7f669efe3577945081f20d4c9c9e78f774a7b2 --- /dev/null +++ b/tests/pytest/handle_taosd_val_log.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' +IN_TDINTERNAL="community" +TDIR=`pwd` +if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../.. +else + cd ../../.. +fi +TOP_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep -v community|grep debug|head -n1` +VALGRIND_OUT=taosd_valgrind.out +VALGRIND_ERR=taosd_valgrind.err +rm -rf /var/lib/taos/* +# nohup valgrind --tool=memcheck --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR & +nohup valgrind --leak-check=yes $TAOSD_DIR > $TDIR/$VALGRIND_OUT 2> $TDIR/$VALGRIND_ERR & +sleep 20 +cd - +./crash_gen.sh -p -t 10 -s 200 +ps -ef |grep valgrind|grep -v grep|awk '{print $2}'|xargs kill -term +while true +do + monitoring=` ps -ef|grep valgrind |grep -v grep| wc -l` + if [ $monitoring -eq 0 ] + then + echo "Manipulator is not running " + break + else + sleep 1 + fi +done + +grep 'start to execute\|ERROR SUMMARY' $VALGRIND_ERR | grep -v 'grep' | uniq | tee taosd_mem_err.log + +for memError in `grep 'ERROR SUMMARY' taosd_mem_err.log | awk '{print $4}'` +do +memError=(${memError//,/}) +if [ -n "$memError" ]; then + if [ "$memError" -gt 12 ]; then + echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ + More than our threshold! ## ${NC}" + fi +fi +done + +grep 'start to execute\|definitely lost:' $VALGRIND_ERR|grep -v 'grep'|uniq|tee taosd-definitely-lost-out.log +for defiMemError in `grep 'definitely lost:' taosd-definitely-lost-out.log | awk '{print $7}'` +do +defiMemError=(${defiMemError//,/}) +if [ -n "$defiMemError" ]; then + if [ "$defiMemError" -gt 0 -a "$defiMemError" -lt 1013 ]; then + cat $VALGRIND_ERR + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 + elif [ "$defiMemError" -gt 1013 ];then #add for azure + cat $VALGRIND_ERR + echo -e "${RED} ## Memory errors number valgrind reports \ + Definitely lost is $defiMemError. More than our threshold! ## ${NC}" + exit 8 + fi +fi +done diff --git a/tests/pytest/insert/basic_unsigned.py b/tests/pytest/insert/basic_unsigned.py new file mode 100644 index 0000000000000000000000000000000000000000..ff7e0e5e4ad57a35d574d88ea179f713c8a8b611 --- /dev/null +++ b/tests/pytest/insert/basic_unsigned.py @@ -0,0 +1,56 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + ret = tdSql.execute( + 'create table tb (ts timestamp, speed tinyint unsigned)') + + insertRows = 10 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into tb values (now + %dm, %d)' % + (i, i)) + + tdLog.info("insert earlier data") + tdSql.execute('insert into tb values (now - 5m , NULL)') + tdSql.execute('insert into tb values (now - 6m , 10)') + tdSql.execute('insert into tb values (now - 7m , NULL)') + tdSql.execute('insert into tb values (now - 8m , 254)') + + tdSql.error('insert into tb values (now - 9m, -1)') + tdSql.error('insert into tb values (now - 9m, 255)') + + tdSql.query("select * from tb") + tdSql.checkRows(insertRows + 4) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/before_1970.py b/tests/pytest/insert/before_1970.py index cb17b657aad1a3bfdb915c9661bad291b75d6f04..b2c4dc57c7a37b4b119308335d326521fe986f08 100644 --- a/tests/pytest/insert/before_1970.py +++ b/tests/pytest/insert/before_1970.py @@ -41,7 +41,7 @@ class TDTestCase: #TODO : should add more testcases tdSql.execute("insert into test values('1930-12-12 01:19:20.345', 1);") tdSql.execute("insert into test values('1969-12-30 23:59:59.999', 2);") - tdSql.execute("insert into test values(-3600, 3);") + tdSql.execute("insert into test values(-3600001, 3);") tdSql.execute("insert into test values('2020-10-20 14:02:53.770', 4);") print("==============insert data") @@ -61,11 +61,11 @@ class TDTestCase: print("==============step4") tdSql.execute("use demo;") tdSql.query("select * from test;") - # print(tdSql.queryResult) + print(tdSql.queryResult) tdSql.checkRows(4) tdSql.checkData(0,0,'1930-12-12 01:19:20.345000') tdSql.checkData(1,0,'1969-12-30 23:59:59.999000') - tdSql.checkData(2,0,'1970-01-01 07:00:00.000000') + tdSql.checkData(2,0,'1970-01-01 06:59:59.999000') tdSql.checkData(3,0,'2020-10-20 14:02:53.770000') print("==============check data") diff --git a/tests/pytest/insert/boundary2.py b/tests/pytest/insert/boundary2.py new file mode 100644 index 0000000000000000000000000000000000000000..99784b7cd9cfa7fc8d32720442cefa7946f6511f --- /dev/null +++ b/tests/pytest/insert/boundary2.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string +import time +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def get_random_string(self, length): + letters = string.ascii_lowercase + result_str = ''.join(random.choice(letters) for i in range(length)) + return result_str + + def run(self): + tdSql.prepare() + + startTime = time.time() + print("==============step1") + sql = "create table stb(ts timestamp, " + for i in range(1022): + sql += "col%d binary(14), " % (i + 1) + sql += "col1023 binary(22))" + tdSql.execute(sql) + + for i in range(4096): + sql = "insert into stb values(%d, " + for j in range(1022): + str = "'%s', " % self.get_random_string(14) + sql += str + sql += "'%s')" % self.get_random_string(22) + tdSql.execute(sql % (self.ts + i)) + + tdSql.query("select * from stb") + tdSql.checkRows(4096) + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdSql.query("select * from stb") + tdSql.checkRows(4096) + + endTime = time.time() + + print("total time %ds" % (endTime - startTime)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/metadataUpdate.py b/tests/pytest/insert/metadataUpdate.py new file mode 100644 index 0000000000000000000000000000000000000000..4c32b0e39ac87bbb5741c77a90f94374fc0e8f5e --- /dev/null +++ b/tests/pytest/insert/metadataUpdate.py @@ -0,0 +1,122 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +from multiprocessing import Process + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + self.tables = 10 + self.rows = 1000 + + def updateMetadata(self): + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = tdDnodes.getSimCfgPath() + + self.conn = taos.connect(host = self.host, user = self.user, password = self.password, config = self.config) + self.cursor = self.conn.cursor() + self.cursor.execute("alter table db.tb add column col2 int") + print("alter table done") + + def deleteTableAndRecreate(self): + self.host = "127.0.0.1" + self.user = "root" + self.password = "taosdata" + self.config = tdDnodes.getSimCfgPath() + + self.conn = taos.connect(host = self.host, user = self.user, password = self.password, config = self.config) + self.cursor = self.conn.cursor() + + self.cursor.execute("use test") + print("drop table stb") + self.cursor.execute("drop table stb") + + print("create table stb") + self.cursor.execute("create table if not exists stb (ts timestamp, col1 int) tags(areaid int, city nchar(20))") + print("insert data") + for i in range(self.tables): + city = "beijing" if i % 2 == 0 else "shanghai" + self.cursor.execute("create table tb%d using stb tags(%d, '%s')" % (i, i, city)) + for j in range(self.rows): + self.cursor.execute("insert into tb%d values(%d, %d)" % (i, self.ts + j, j * 100000)) + + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table if not exists tb (ts timestamp, col1 int)") + tdSql.execute("insert into tb values(%d, 1)" % self.ts) + + print("==============step2") + tdSql.query("select * from tb") + tdSql.checkRows(1) + + p = Process(target=self.updateMetadata, args=()) + p.start() + p.join() + p.terminate() + + tdSql.execute("insert into tb(ts, col1, col2) values(%d, 1, 2)" % (self.ts + 2)) + + print("==============step2") + tdSql.query("select * from tb") + tdSql.checkRows(2) + + # Add test case: https://jira.taosdata.com:18080/browse/TD-3474 + + print("==============step1") + tdSql.execute("create database test") + tdSql.execute("use test") + tdSql.execute("create table if not exists stb (ts timestamp, col1 int) tags(areaid int, city nchar(20))") + + for i in range(self.tables): + city = "beijing" if i % 2 == 0 else "shanghai" + tdSql.execute("create table tb%d using stb tags(%d, '%s')" % (i, i, city)) + for j in range(self.rows): + tdSql.execute("insert into tb%d values(%d, %d)" % (i, self.ts + j, j * 100000)) + + tdSql.query("select count(*) from stb") + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(*) from tb1") + tdSql.checkData(0, 0, 1000) + + p = Process(target=self.deleteTableAndRecreate, args=()) + p.start() + p.join() + p.terminate() + + tdSql.query("select count(*) from stb") + tdSql.checkData(0, 0, 10000) + + tdSql.query("select count(*) from tb1") + tdSql.checkData(0, 0, 1000) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/retentionpolicy.py b/tests/pytest/insert/retentionpolicy.py index c69060b5aec16b8d58aa2742e24c44d1ebe543ab..e0446113d6d1fb197490a09ebd1ebe4b5b12e66f 100644 --- a/tests/pytest/insert/retentionpolicy.py +++ b/tests/pytest/insert/retentionpolicy.py @@ -44,7 +44,8 @@ class TDTestRetetion: caller = inspect.getframeinfo(inspect.stack()[1][0]) args = (caller.filename, caller.lineno, sql, self.queryRows, expectRows) os.system("sudo timedatectl set-ntp true") - time.sleep(40) + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1))) + time.sleep(5) tdLog.exit("%s(%d) failed: sql:%s, queryRows:%d != expect:%d" % args) def run(self): @@ -63,7 +64,7 @@ class TDTestRetetion: tdLog.info("=============== step2") tdDnodes.stop(1) os.system("sudo timedatectl set-ntp false") - os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48))) tdDnodes.start(1) cmd = 'insert into test values(now,5);' tdDnodes.stop(1) @@ -79,7 +80,7 @@ class TDTestRetetion: self.checkRows(5,cmd) tdLog.info("=============== step3") tdDnodes.stop(1) - os.system("sudo date -s $(date -d \"${DATE} 2 days\" \"+%Y%m%d\")") + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=48))) tdDnodes.start(1) tdLog.info(cmd) tdSql.execute(cmd) @@ -99,18 +100,19 @@ class TDTestRetetion: tdLog.info(cmd) tdSql.execute(cmd) self.queryRows=tdSql.query('select * from test') - self.checkRows(7,cmd) + self.checkRows(5,cmd) tdLog.info("=============== step5") tdDnodes.stop(1) tdDnodes.start(1) cmd='select * from test where ts > now-1d' self.queryRows=tdSql.query('select * from test where ts > now-1d') - self.checkRows(1,cmd) + self.checkRows(2,cmd) def stop(self): os.system("sudo timedatectl set-ntp true") - time.sleep(40) + os.system("date -s '%s'"%(datetime.datetime.now()+datetime.timedelta(hours=1))) + time.sleep(5) tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/insert/unsignedBigint.py b/tests/pytest/insert/unsignedBigint.py new file mode 100644 index 0000000000000000000000000000000000000000..b222f2cd0195e14fddf1f10662447cef97f0e841 --- /dev/null +++ b/tests/pytest/insert/unsignedBigint.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed bigint unsigned)') + tdSql.execute('create table tb (ts timestamp, speed bigint unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 18446744073709551614)") + tdSql.execute("insert into tb values (now+2m, 18446744073709551614)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 18446744073709551614)') + tdSql.checkData(0, 1, 18446744073709551614) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 18446744073709551615) -x step4") + tdSql.error("insert into tb values (now+3m, 18446744073709551615)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now-4m, -1)") + tdSql.error("insert into tb values (now-4m, -1)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/unsignedInt.py b/tests/pytest/insert/unsignedInt.py new file mode 100644 index 0000000000000000000000000000000000000000..ed18999bc415022da34788a79e9045a5f67cf8ee --- /dev/null +++ b/tests/pytest/insert/unsignedInt.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed int unsigned)') + tdSql.execute('create table tb (ts timestamp, speed int unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 4294967294)") + tdSql.execute("insert into tb values (now+2m, 4294967294)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 4294967294)') + tdSql.checkData(0, 1, 4294967294) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 4294967295) -x step4") + tdSql.error("insert into tb values (now+3m, 4294967295)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now-4m, -1)") + tdSql.error("insert into tb values (now-4m, -1)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/unsignedSmallint.py b/tests/pytest/insert/unsignedSmallint.py new file mode 100644 index 0000000000000000000000000000000000000000..9893c470ce1a53a05138dd9d5c18e2b0a9f21374 --- /dev/null +++ b/tests/pytest/insert/unsignedSmallint.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed smallint unsigned)') + tdSql.execute('create table tb (ts timestamp, speed smallint unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 65534)") + tdSql.execute("insert into tb values (now+2m, 65534)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 65534)') + tdSql.checkData(0, 1, 65534) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 65535) -x step4") + tdSql.error("insert into tb values (now+3m, 65535)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now-4m, -1)") + tdSql.error("insert into tb values (now-4m, -1)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) +# convert end + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/unsignedTinyint.py b/tests/pytest/insert/unsignedTinyint.py new file mode 100644 index 0000000000000000000000000000000000000000..5bdfe7580b198ccaffcab7005e88aa72e1c437af --- /dev/null +++ b/tests/pytest/insert/unsignedTinyint.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('=============== step1') + tdLog.info('create table tb (ts timestamp, speed tinyint unsigned)') + tdSql.execute('create table tb (ts timestamp, speed tinyint unsigned)') + tdLog.info("insert into tb values (now, NULL)") + tdSql.execute("insert into tb values (now, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step2') + tdLog.info("insert into tb values (now+1m, -1) -x step2") + tdSql.error("insert into tb values (now+1m, -1) ") + tdLog.info("insert into tb values (now+1m, NULL)") + tdSql.execute("insert into tb values (now+1m, NULL)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(2)') + tdSql.checkRows(2) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step3') + tdLog.info("insert into tb values (now+2m, 254)") + tdSql.execute("insert into tb values (now+2m, 254)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(3)') + tdSql.checkRows(3) + tdLog.info('tdSql.checkData(0, 1, 254)') + tdSql.checkData(0, 1, 254) + tdLog.info('=============== step4') + tdLog.info("insert into tb values (now+3m, 255) -x step4") + tdSql.error("insert into tb values (now+3m, 255)") + tdLog.info("insert into tb values (now+3m, NULL)") + tdSql.execute("insert into tb values (now+3m, NULL)") + tdLog.info('select * from tb') + tdSql.query('select * from tb') + tdLog.info('tdSql.checkRow(4)') + tdSql.checkRows(4) + tdLog.info('tdSql.checkData(0, 1, null)') + tdSql.checkData(0, 1, None) + tdLog.info('=============== step5') + tdLog.info("insert into tb values (now+4m, a2)") + tdSql.error("insert into tb values (now+4m, a2)") + tdLog.info("insert into tb values (now+4m, 0)") + tdSql.execute("insert into tb values (now+4m, 0)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + tdLog.info('tdSql.checkData(0, 1, 0)') + tdSql.checkData(0, 1, 0) + tdLog.info('=============== step6') + tdLog.info("insert into tb values (now+5m, 2a)") + tdSql.error("insert into tb values (now+5m, 2a)") + tdLog.info("insert into tb values (now+5m, 2)") + tdSql.execute("insert into tb values (now+5m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(6)') + tdSql.checkRows(6) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('=============== step7') + tdLog.info("insert into tb values (now+6m, 2a'1)") + tdSql.error("insert into tb values (now+6m, 2a'1)") + tdLog.info("insert into tb values (now+6m, 2)") + tdSql.execute("insert into tb values (now+6m, 2)") + tdLog.info('select * from tb order by ts desc') + tdSql.query('select * from tb order by ts desc') + tdLog.info('tdSql.checkRow(7)') + tdSql.checkRows(7) + tdLog.info('tdSql.checkData(0, 1, 2)') + tdSql.checkData(0, 1, 2) + tdLog.info('drop database db') + tdSql.execute('drop database db') + tdLog.info('show databases') + tdSql.query('show databases') + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/multilevel/fileDistributionSameLevel.py b/tests/pytest/multilevel/fileDistributionSameLevel.py index e2880283b5b6a76942ac9ac5abba50f8c5bfda1e..83ce85671771c4d2627541e80262ef8c764e45c3 100644 --- a/tests/pytest/multilevel/fileDistributionSameLevel.py +++ b/tests/pytest/multilevel/fileDistributionSameLevel.py @@ -18,26 +18,44 @@ from util.cases import * from util.sql import * from util.dnodes import * - +dataDir = ['data00','data01','data02','data03','data04'] +dataDict = {'data00':0,'data01':0,'data02':0,'data03':0,'data04':0} class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) + def getfiles(self,ospath): + try: + files = os.listdir(ospath) + for f in files: + path = os.path.join(ospath, f) + if os.path.isfile(path): + if path.endswith('.data'): + for i in dataDir: + if i in path: + dataDict[i] = dataDict[i] + 1 + print(path) + if os.path.isdir(path): + self.getfiles(path) + except Exception as e : + print(str(e)) + def run(self): - self.ntables = 10 - self.rowsPerTable = 10 + self.ntables = 1000 self.ts = 1520000010000 tdDnodes.stop(1) # Test1 1 dataDir + tdLog.info("================= step1") cfg={ '10' : 'maxVgroupsPerDb', + '100' : 'maxTablesPerVnode', '/mnt/data00 0 1' : 'dataDir', '/mnt/data01 0 0' : 'dataDir', '/mnt/data02 0 0' : 'dataDir', - '/mnt/data03 1 0' : 'dataDir', - '/mnt/data04 1 0' : 'dataDir' + '/mnt/data03 0 0' : 'dataDir', + '/mnt/data04 0 0' : 'dataDir' } tdSql.createDir('/mnt/data00') tdSql.createDir('/mnt/data01') @@ -51,18 +69,37 @@ class TDTestCase: tdSql.execute("create database test days 1") tdSql.execute("use test") - tdSql.execute("create table tb(ts timestamp, c int)") + tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") - for i in range(self.rowsPerTable): - tdSql.execute("insert into tb values(%d, 1)" % (self.ts + i * 86400000)) - + for i in range(self.ntables): + tdSql.execute("create table tb%d using stb tags(%d)" %(i, i)) + tdSql.execute("insert into tb%d values(%d, 1)" % (i,self.ts + int (i / 100) * 86400000)) + + tdLog.info("================= step2") tdDnodes.stop(1) tdDnodes.start(1) - tdSql.query("select * from test.tb") - tdSql.checkRows(10) - + tdSql.query("select * from test.stb") + tdSql.checkRows(1000) + tdLog.info("================= step3") + tdSql.execute('drop database test') + for i in range(50): + tdSql.execute("create database test%d days 1" %(i)) + tdSql.execute("use test%d" %(i)) + tdSql.execute("create table tb (ts timestamp,i int)") + for j in range(10): + tdSql.execute("insert into tb values(%d, 1)" % (self.ts + int (i / 100) * 86400000)) + tdDnodes.stop(1) + tdDnodes.start(1) + flag = True + for i in range(4): + if dataDict[dataDir[i]] == dataDict[dataDir[i+1]]: + flag = flag & True + else: + flag = flag & False + break + if not flag : tdLog.exit("%s failed, expect not occured" % (sys.argv[0])) def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/multilevel/threeLevelMountPoint.py b/tests/pytest/multilevel/threeLevelMountPoint.py index ccaf95f47d389e053c042a4a20cefad2c7d08776..d97602eb76889ab265f6bc673f5c7558be5d7fe3 100644 --- a/tests/pytest/multilevel/threeLevelMountPoint.py +++ b/tests/pytest/multilevel/threeLevelMountPoint.py @@ -56,6 +56,15 @@ class TDTestCase: tdDnodes.startWithoutSleep(1) tdSql.taosdStatus(1) + tdSql.haveFile('/mnt/data00',1) + tdSql.haveFile('/mnt/data01',1) + tdSql.haveFile('/mnt/data02',1) + tdSql.haveFile('/mnt/data10',1) + tdSql.haveFile('/mnt/data11',1) + tdSql.haveFile('/mnt/data12',1) + tdSql.haveFile('/mnt/data20',1) + tdSql.haveFile('/mnt/data21',1) + tdSql.haveFile('/mnt/data22',1) def stop(self): tdSql.close() diff --git a/tests/pytest/perfbenchmark/bug3433.py b/tests/pytest/perfbenchmark/bug3433.py new file mode 100644 index 0000000000000000000000000000000000000000..80e9cce0dc60937b3eaf7c122a8ae5951fd1ad6b --- /dev/null +++ b/tests/pytest/perfbenchmark/bug3433.py @@ -0,0 +1,245 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import json + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def __init__(self): + self.path = "" + + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def getcfgPath(self, path): + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug(f"binPath {binPath}") + binPath = os.path.realpath(binPath) + tdLog.debug(f"binPath real path {binPath}") + if path == "": + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + return self.path + + def getCfgDir(self): + self.getcfgPath(self.path) + self.cfgDir = f"{self.path}/sim/psim/cfg" + return self.cfgDir + + def creatcfg(self): + dbinfo = { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 3650, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "cachelast": 0, + "quorum": 1, + "fsync": 3000, + "update": 0 + } + + # set stable schema + stable1 = { + "name": "stb1", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "t1", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 1, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 20000, + "start_timestamp": "2020-12-31 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + + ], + "tags": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + ] + } + stable2 = { + "name": "stb2", + "child_table_exists": "no", + "childtable_count": 1000, + "childtable_prefix": "t2", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 1, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 20000, + "start_timestamp": "2020-12-31 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + + ], + "tags": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + ] + } + + # create different stables like stable1 and add to list super_tables + super_tables = [] + super_tables.append(stable1) + super_tables.append(stable2) + database = { + "dbinfo": dbinfo, + "super_tables": super_tables + } + + cfgdir = self.getCfgDir() + create_table = { + "filetype": "insert", + "cfgdir": cfgdir, + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "/tmp/insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "databases": [database] + } + return create_table + + def createinsertfile(self): + create_table = self.creatcfg() + date = datetime.datetime.now().strftime("%Y%m%d%H%M") + file_create_table = f"/tmp/insert_{date}.json" + + with open(file_create_table, 'w') as f: + json.dump(create_table, f) + return file_create_table + + def inserttable(self, filepath): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + binPath = buildPath + "/build/bin/" + + create_table_cmd = f"{binPath}taosdemo -f {filepath} > /dev/null 2>&1" + _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") + + def droptmpfile(self): + drop_file_cmd = "rm -f /tmp/insert_* " + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + + def run(self): + tdLog.printNoPrefix("==========step1:create database and insert records") + file_create_table = self.createinsertfile() + self.inserttable(file_create_table) + + tdLog.printNoPrefix("==========step2:check database and stable records") + tdSql.query("show databases") + tdSql.checkData(0, 2, 2000) + tdSql.execute("use db") + tdSql.query("show stables") + tdSql.checkData(0, 4, 1000) + tdSql.checkData(1, 4, 1000) + + self.droptmpfile() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/pytest/perfbenchmark/bug3589.py b/tests/pytest/perfbenchmark/bug3589.py new file mode 100644 index 0000000000000000000000000000000000000000..c54ef8595d0bea1c4984b3f90e282f09659576c3 --- /dev/null +++ b/tests/pytest/perfbenchmark/bug3589.py @@ -0,0 +1,156 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import json + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def __init__(self): + self.path = "" + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/debug/build/bin")] + break + return buildPath + + def getCfgDir(self): + return self.getBuildPath() + "/sim/psim/cfg" + + def querycfg(self): + cfgdir = self.getCfgDir() + querycfg={ + "filetype": "query", + "cfgdir": cfgdir, + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "yes", + "databases": "db", + "specified_table_query": { + "query_interval": 0, + "concurrent": 1, + "sqls": [ + { + "sql": "select * from t10, t11 where t10.ts=t11.ts" + } + ] + } + } + + return querycfg + + def querycfgfile(self): + querycfg = self.querycfg() + date = datetime.datetime.now().strftime("%Y%m%d%H%M") + querycfg.get("specified_table_query").get("sqls")[0]["result"] = f"/tmp/query_{date}.log" + file_query_table = f"/tmp/query_{date}.json" + with open(file_query_table, "w") as f: + json.dump(querycfg, f) + + return [file_query_table, querycfg] + + def querytable(self, filepath): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + binPath = buildPath + "/debug/build/bin/" + + query_table_cmd = f"yes | {binPath}taosdemo -f {filepath}" + _ = subprocess.check_output(query_table_cmd, shell=True).decode("utf-8") + + def checkqueryresult(self, expectrows): + querycfg = self.querycfgfile()[1] + result_file = querycfg.get("specified_table_query").get("sqls")[0].get("result") + "-0" + if result_file: + check_cmd = f"wc -l {result_file}" + check_data_init = subprocess.check_output(check_cmd, shell=True).decode("utf-8") + check_data = int(check_data_init[0]) + if check_data == expectrows: + tdLog.info(f"queryResultRows:{check_data} == expect:{expectrows}") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, check_data, expectrows) + tdLog.exit(f"{args[0]}({args[1]}) failed: result:{args[2]} != expect:{args[3]}") + + def droptmpfile(self): + drop_file_cmd = "rm -f /tmp/query_* " + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + drop_file_cmd = "rm -f querySystemInfo-*" + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table && insert data") + tdSql.execute("alter database db keep 36500") + tdSql.execute( + "create table stb1 (ts timestamp, c1 int) TAGS(t1 int)" + ) + tdSql.execute("create table t10 using stb1 tags(1)") + tdSql.execute("create table t11 using stb1 tags(2)") + + tdSql.execute("insert into t10 values (-865000000, 1)") + tdSql.execute("insert into t11 values (-865000000, 2)") + tdSql.execute("insert into t10 values ('1969-12-31 23:59:59.000', 2)") + tdSql.execute("insert into t11 values ('1969-12-31 23:59:59.000', 3)") + tdSql.execute("insert into t10 values ('1970-01-01 00:00:00.000', 3)") + tdSql.execute("insert into t11 values ('1970-01-01 00:00:00.000', 4)") + tdSql.execute("insert into t10 values (-15230000, 4)") + tdSql.execute("insert into t11 values (-15230000, 5)") + tdSql.execute("insert into t10 values (-15220000, 5)") + tdSql.execute("insert into t11 values (-15220000, 6)") + tdSql.execute("insert into t10 values (-15210000, 6)") + tdSql.execute("insert into t11 values (-15210000, 7)") + tdSql.execute("insert into t10 values (0, 7)") + tdSql.execute("insert into t11 values (0, 8)") + tdSql.execute("insert into t10 values ('2020-10-01 00:00:00.000', 8)") + tdSql.execute("insert into t11 values ('2020-10-01 00:00:00.000', 9)") + + tdLog.printNoPrefix("==========step2:query") + query_file = self.querycfgfile()[0] + self.querytable(query_file) + self.checkqueryresult(8) + + self.droptmpfile() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh deleted file mode 100755 index ad26c480048e04e8a0955f908c0396d63726513e..0000000000000000000000000000000000000000 --- a/tests/pytest/pytest_1.sh +++ /dev/null @@ -1,240 +0,0 @@ -#!/bin/bash -ulimit -c unlimited - -python3 ./test.py -f insert/basic.py -python3 ./test.py -f insert/int.py -python3 ./test.py -f insert/float.py -python3 ./test.py -f insert/bigint.py -python3 ./test.py -f insert/bool.py -python3 ./test.py -f insert/double.py -python3 ./test.py -f insert/smallint.py -python3 ./test.py -f insert/tinyint.py -python3 ./test.py -f insert/date.py -python3 ./test.py -f insert/binary.py -python3 ./test.py -f insert/nchar.py -#python3 ./test.py -f insert/nchar-boundary.py -python3 ./test.py -f insert/nchar-unicode.py -python3 ./test.py -f insert/multi.py -python3 ./test.py -f insert/randomNullCommit.py -python3 insert/retentionpolicy.py -python3 ./test.py -f insert/alterTableAndInsert.py -python3 ./test.py -f insert/insertIntoTwoTables.py -#python3 ./test.py -f insert/before_1970.py -python3 bug2265.py - -#table -python3 ./test.py -f table/alter_wal0.py -python3 ./test.py -f table/column_name.py -python3 ./test.py -f table/column_num.py -python3 ./test.py -f table/db_table.py -python3 ./test.py -f table/create_sensitive.py -#python3 ./test.py -f table/tablename-boundary.py -python3 ./test.py -f table/max_table_length.py -python3 ./test.py -f table/alter_column.py -python3 ./test.py -f table/boundary.py -python3 ./test.py -f table/create.py -python3 ./test.py -f table/del_stable.py - - -# tag -python3 ./test.py -f tag_lite/filter.py -python3 ./test.py -f tag_lite/create-tags-boundary.py -python3 ./test.py -f tag_lite/3.py -python3 ./test.py -f tag_lite/4.py -python3 ./test.py -f tag_lite/5.py -python3 ./test.py -f tag_lite/6.py -python3 ./test.py -f tag_lite/add.py -python3 ./test.py -f tag_lite/bigint.py -python3 ./test.py -f tag_lite/binary_binary.py -python3 ./test.py -f tag_lite/binary.py -python3 ./test.py -f tag_lite/bool_binary.py -python3 ./test.py -f tag_lite/bool_int.py -python3 ./test.py -f tag_lite/bool.py -python3 ./test.py -f tag_lite/change.py -python3 ./test.py -f tag_lite/column.py -python3 ./test.py -f tag_lite/commit.py -python3 ./test.py -f tag_lite/create.py -python3 ./test.py -f tag_lite/datatype.py -python3 ./test.py -f tag_lite/datatype-without-alter.py -python3 ./test.py -f tag_lite/delete.py -python3 ./test.py -f tag_lite/double.py -python3 ./test.py -f tag_lite/float.py -python3 ./test.py -f tag_lite/int_binary.py -python3 ./test.py -f tag_lite/int_float.py -python3 ./test.py -f tag_lite/int.py -python3 ./test.py -f tag_lite/set.py -python3 ./test.py -f tag_lite/smallint.py -python3 ./test.py -f tag_lite/tinyint.py - -#python3 ./test.py -f dbmgmt/database-name-boundary.py - -python3 ./test.py -f import_merge/importBlock1HO.py -python3 ./test.py -f import_merge/importBlock1HPO.py -python3 ./test.py -f import_merge/importBlock1H.py -python3 ./test.py -f import_merge/importBlock1S.py -python3 ./test.py -f import_merge/importBlock1Sub.py -python3 ./test.py -f import_merge/importBlock1TO.py -python3 ./test.py -f import_merge/importBlock1TPO.py -python3 ./test.py -f import_merge/importBlock1T.py -python3 ./test.py -f import_merge/importBlock2HO.py -python3 ./test.py -f import_merge/importBlock2HPO.py -python3 ./test.py -f import_merge/importBlock2H.py -python3 ./test.py -f import_merge/importBlock2S.py -python3 ./test.py -f import_merge/importBlock2Sub.py -python3 ./test.py -f import_merge/importBlock2TO.py -python3 ./test.py -f import_merge/importBlock2TPO.py -python3 ./test.py -f import_merge/importBlock2T.py -python3 ./test.py -f import_merge/importBlockbetween.py -python3 ./test.py -f import_merge/importCacheFileHO.py -python3 ./test.py -f import_merge/importCacheFileHPO.py -python3 ./test.py -f import_merge/importCacheFileH.py -python3 ./test.py -f import_merge/importCacheFileS.py -python3 ./test.py -f import_merge/importCacheFileSub.py -python3 ./test.py -f import_merge/importCacheFileTO.py -python3 ./test.py -f import_merge/importCacheFileTPO.py -python3 ./test.py -f import_merge/importCacheFileT.py -python3 ./test.py -f import_merge/importDataH2.py -python3 ./test.py -f import_merge/importDataHO2.py -python3 ./test.py -f import_merge/importDataHO.py -python3 ./test.py -f import_merge/importDataHPO.py -python3 ./test.py -f import_merge/importDataLastHO.py -python3 ./test.py -f import_merge/importDataLastHPO.py -python3 ./test.py -f import_merge/importDataLastH.py -python3 ./test.py -f import_merge/importDataLastS.py -python3 ./test.py -f import_merge/importDataLastSub.py -python3 ./test.py -f import_merge/importDataLastTO.py -python3 ./test.py -f import_merge/importDataLastTPO.py -python3 ./test.py -f import_merge/importDataLastT.py -python3 ./test.py -f import_merge/importDataS.py -python3 ./test.py -f import_merge/importDataSub.py -python3 ./test.py -f import_merge/importDataTO.py -python3 ./test.py -f import_merge/importDataTPO.py -python3 ./test.py -f import_merge/importDataT.py -python3 ./test.py -f import_merge/importHeadOverlap.py -python3 ./test.py -f import_merge/importHeadPartOverlap.py -python3 ./test.py -f import_merge/importHead.py -python3 ./test.py -f import_merge/importHORestart.py -python3 ./test.py -f import_merge/importHPORestart.py -python3 ./test.py -f import_merge/importHRestart.py -python3 ./test.py -f import_merge/importLastHO.py -python3 ./test.py -f import_merge/importLastHPO.py -python3 ./test.py -f import_merge/importLastH.py -python3 ./test.py -f import_merge/importLastS.py -python3 ./test.py -f import_merge/importLastSub.py -python3 ./test.py -f import_merge/importLastTO.py -python3 ./test.py -f import_merge/importLastTPO.py -python3 ./test.py -f import_merge/importLastT.py -python3 ./test.py -f import_merge/importSpan.py -python3 ./test.py -f import_merge/importSRestart.py -python3 ./test.py -f import_merge/importSubRestart.py -python3 ./test.py -f import_merge/importTailOverlap.py -python3 ./test.py -f import_merge/importTailPartOverlap.py -python3 ./test.py -f import_merge/importTail.py -python3 ./test.py -f import_merge/importToCommit.py -python3 ./test.py -f import_merge/importTORestart.py -python3 ./test.py -f import_merge/importTPORestart.py -python3 ./test.py -f import_merge/importTRestart.py -python3 ./test.py -f import_merge/importInsertThenImport.py -python3 ./test.py -f import_merge/importCSV.py -# user -python3 ./test.py -f user/user_create.py -python3 ./test.py -f user/pass_len.py - -# stable -python3 ./test.py -f stable/query_after_reset.py - -#query -python3 ./test.py -f query/filter.py -python3 ./test.py -f query/filterCombo.py -python3 ./test.py -f query/queryNormal.py -python3 ./test.py -f query/queryError.py -python3 ./test.py -f query/filterAllIntTypes.py -python3 ./test.py -f query/filterFloatAndDouble.py -python3 ./test.py -f query/filterOtherTypes.py -python3 ./test.py -f query/querySort.py -python3 ./test.py -f query/queryJoin.py -python3 ./test.py -f query/select_last_crash.py -python3 ./test.py -f query/queryNullValueTest.py -python3 ./test.py -f query/queryInsertValue.py -python3 ./test.py -f query/queryConnection.py -python3 ./test.py -f query/queryCountCSVData.py -python3 ./test.py -f query/natualInterval.py -python3 ./test.py -f query/bug1471.py -#python3 ./test.py -f query/dataLossTest.py -python3 ./test.py -f query/bug1874.py -python3 ./test.py -f query/bug1875.py -python3 ./test.py -f query/bug1876.py -python3 ./test.py -f query/bug2218.py -python3 ./test.py -f query/bug2117.py -python3 ./test.py -f query/bug2118.py -python3 ./test.py -f query/bug2143.py -python3 ./test.py -f query/sliding.py -python3 ./test.py -f query/unionAllTest.py -python3 ./test.py -f query/bug2281.py -python3 ./test.py -f query/bug2119.py -python3 ./test.py -f query/isNullTest.py -python3 ./test.py -f query/queryWithTaosdKilled.py -python3 ./test.py -f query/floatCompare.py - -#stream -python3 ./test.py -f stream/metric_1.py -python3 ./test.py -f stream/metric_n.py -python3 ./test.py -f stream/new.py -python3 ./test.py -f stream/stream1.py -python3 ./test.py -f stream/stream2.py -#python3 ./test.py -f stream/parser.py -python3 ./test.py -f stream/history.py -python3 ./test.py -f stream/sys.py -python3 ./test.py -f stream/table_1.py -python3 ./test.py -f stream/table_n.py - -#alter table -python3 ./test.py -f alter/alter_table_crash.py - -# client -python3 ./test.py -f client/client.py -python3 ./test.py -f client/version.py -python3 ./test.py -f client/alterDatabase.py -python3 ./test.py -f client/noConnectionErrorTest.py - -# Misc -python3 testCompress.py -python3 testNoCompress.py -python3 testMinTablesPerVnode.py - -# functions -python3 ./test.py -f functions/function_avg.py -r 1 -python3 ./test.py -f functions/function_bottom.py -r 1 -python3 ./test.py -f functions/function_count.py -r 1 -python3 ./test.py -f functions/function_diff.py -r 1 -python3 ./test.py -f functions/function_first.py -r 1 -python3 ./test.py -f functions/function_last.py -r 1 -python3 ./test.py -f functions/function_last_row.py -r 1 -python3 ./test.py -f functions/function_leastsquares.py -r 1 -python3 ./test.py -f functions/function_max.py -r 1 -python3 ./test.py -f functions/function_min.py -r 1 -python3 ./test.py -f functions/function_operations.py -r 1 -python3 ./test.py -f functions/function_percentile.py -r 1 -python3 ./test.py -f functions/function_spread.py -r 1 -python3 ./test.py -f functions/function_stddev.py -r 1 -python3 ./test.py -f functions/function_sum.py -r 1 -python3 ./test.py -f functions/function_top.py -r 1 -python3 ./test.py -f functions/function_twa.py -r 1 -python3 ./test.py -f functions/function_twa_test2.py -python3 queryCount.py -python3 ./test.py -f query/queryGroupbyWithInterval.py -python3 client/twoClients.py -python3 test.py -f query/queryInterval.py -python3 test.py -f query/queryFillTest.py - -# tools -python3 test.py -f tools/taosdemoTest.py -python3 test.py -f tools/taosdumpTest.py -python3 test.py -f tools/lowaTest.py -python3 test.py -f tools/taosdemoTest2.py - -# subscribe -python3 test.py -f subscribe/singlemeter.py -#python3 test.py -f subscribe/stability.py -python3 test.py -f subscribe/supertable.py - diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh deleted file mode 100755 index 4ec517a0bf1c5eff8ad670cf28ab63d5ce818460..0000000000000000000000000000000000000000 --- a/tests/pytest/pytest_2.sh +++ /dev/null @@ -1,21 +0,0 @@ - - -# update -python3 ./test.py -f update/allow_update.py -python3 ./test.py -f update/allow_update-0.py -python3 ./test.py -f update/append_commit_data.py -python3 ./test.py -f update/append_commit_last-0.py -python3 ./test.py -f update/append_commit_last.py -python3 ./test.py -f update/merge_commit_data.py -python3 ./test.py -f update/merge_commit_data-0.py -python3 ./test.py -f update/merge_commit_data2.py -python3 ./test.py -f update/merge_commit_data2_update0.py -python3 ./test.py -f update/merge_commit_last-0.py -python3 ./test.py -f update/merge_commit_last.py -python3 ./test.py -f update/bug_td2279.py - -# wal -python3 ./test.py -f wal/addOldWalTest.py - -# function -python3 ./test.py -f functions/all_null_value.py \ No newline at end of file diff --git a/tests/pytest/query/bug3351.py b/tests/pytest/query/bug3351.py new file mode 100644 index 0000000000000000000000000000000000000000..288d071a69cac30e9e5666b4d39a8de5c29f91d9 --- /dev/null +++ b/tests/pytest/query/bug3351.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 36500") + tdSql.execute("use db") + tdLog.printNoPrefix("==========step1:create table && insert data") + + tdSql.execute( + "create table stb1 (ts timestamp, c1 int) TAGS(t1 int)" + ) + tdSql.execute("create table t0 using stb1 tags(1)") + tdSql.execute("insert into t0 values (-865000000, 1)") + tdSql.execute("insert into t0 values (-864000000, 2)") + tdSql.execute("insert into t0 values (-863000000, 3)") + tdSql.execute("insert into t0 values (-15230000, 4)") + tdSql.execute("insert into t0 values (-15220000, 5)") + tdSql.execute("insert into t0 values (-15210000, 6)") + + tdLog.printNoPrefix("==========step2:query") + # bug1:when ts > -864000000, return 0 rows; + # bug2:when ts = -15220000, return 0 rows. + tdSql.query('select * from t0 where ts < -864000000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts <= -864000000') + tdSql.checkRows(2) + tdSql.query('select * from t0 where ts = -864000000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts > -864000000') + tdSql.checkRows(4) + tdSql.query('select * from t0 where ts >= -864000000') + tdSql.checkRows(5) + tdSql.query('select * from t0 where ts < -15220000') + tdSql.checkRows(4) + tdSql.query('select * from t0 where ts <= -15220000') + tdSql.checkRows(5) + tdSql.query('select * from t0 where ts = -15220000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts > -15220000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts >= -15220000') + tdSql.checkRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/bug3375.py b/tests/pytest/query/bug3375.py new file mode 100644 index 0000000000000000000000000000000000000000..25c3467c0508073545c630c2a0fe9d5cc31f5dbe --- /dev/null +++ b/tests/pytest/query/bug3375.py @@ -0,0 +1,61 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 36500") + tdSql.execute("use db") + + tdLog.printNoPrefix("==========step1:create table && insert data") + tdSql.execute( + "create table stb1 (ts timestamp, c11 int) TAGS(t11 int, t12 int )" + ) + tdSql.execute( + "create table stb2 (ts timestamp, c21 int) TAGS(t21 int, t22 int )" + ) + tdSql.execute("create table t10 using stb1 tags(1, 10)") + tdSql.execute("create table t20 using stb2 tags(1, 12)") + tdSql.execute("insert into t10 values (1600000000000, 1)") + tdSql.execute("insert into t10 values (1610000000000, 2)") + tdSql.execute("insert into t20 values (1600000000000, 3)") + tdSql.execute("insert into t20 values (1610000000000, 4)") + + tdLog.printNoPrefix("==========step2:query crash test") + tdSql.query("select stb1.c11, stb1.t11, stb1.t12 from stb2,stb1 where stb2.t21 = stb1.t11 and stb1.ts = stb2.ts") + tdSql.checkRows(2) + tdSql.query("select stb2.c21, stb2.t21, stb2.t21 from stb1, stb2 where stb2.t21 = stb1.t11 and stb1.ts = stb2.ts") + tdSql.checkRows(2) + tdSql.query("select top(stb2.c21,2) from stb1, stb2 where stb2.t21 = stb1.t11 and stb1.ts = stb2.ts") + tdSql.checkRows(2) + tdSql.query("select last(stb2.c21) from stb1, stb2 where stb2.t21 = stb1.t11 and stb1.ts = stb2.ts") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/filter.py b/tests/pytest/query/filter.py index f107985f150a05cdd38a7fa1453d59d35c8a903a..6d2ffbc8b10992628fec381e60fd11a68c69fe27 100644 --- a/tests/pytest/query/filter.py +++ b/tests/pytest/query/filter.py @@ -28,18 +28,18 @@ class TDTestCase: print("==============step1") tdSql.execute( - "create table if not exists st (ts timestamp, tagtype int, name nchar(16)) tags(dev nchar(50))") + "create table if not exists st (ts timestamp, tagtype int, name nchar(16), col4 binary(16)) tags(dev nchar(50), tag2 binary(16))") tdSql.execute( - 'CREATE TABLE if not exists dev_001 using st tags("dev_01")') + 'CREATE TABLE if not exists dev_001 using st tags("dev_01", "tag_01")') tdSql.execute( - 'CREATE TABLE if not exists dev_002 using st tags("dev_02")') + 'CREATE TABLE if not exists dev_002 using st tags("dev_02", "tag_02")') print("==============step2") tdSql.execute( - """INSERT INTO dev_001(ts, tagtype, name) VALUES('2020-05-13 10:00:00.000', 1, 'first'),('2020-05-13 10:00:00.001', 2, 'second'), - ('2020-05-13 10:00:00.002', 3, 'third') dev_002 VALUES('2020-05-13 10:00:00.003', 1, 'first'), ('2020-05-13 10:00:00.004', 2, 'second'), - ('2020-05-13 10:00:00.005', 3, 'third')""") + """INSERT INTO dev_001 VALUES('2020-05-13 10:00:00.000', 1, 'first', 'binary1'),('2020-05-13 10:00:00.001', 2, 'second', 'binary2'), + ('2020-05-13 10:00:00.002', 3, 'third' , 'binary3') dev_002 VALUES('2020-05-13 10:00:00.003', 1, 'first', 'binary4'), ('2020-05-13 10:00:00.004', 2, 'second', 'binary5'), + ('2020-05-13 10:00:00.005', 3, 'third', 'binary6')""") # > for timestamp type tdSql.query("select * from db.st where ts > '2020-05-13 10:00:00.002'") @@ -85,6 +85,12 @@ class TDTestCase: tdSql.query("select * from db.st where name = 'first'") tdSql.checkRows(2) + tdSql.query("select * from db.st where col4 = 1231231") + tdSql.checkRows(0) + + tdSql.query("select * from db.st where name = 1231231") + tdSql.checkRows(0) + # <> for timestamp type tdSql.query("select * from db.st where ts <> '2020-05-13 10:00:00.002'") # tdSql.checkRows(4) @@ -105,6 +111,13 @@ class TDTestCase: tdSql.query("select * from db.st where name like '_econd'") tdSql.checkRows(2) + # for tag + tdSql.query("select * from db.st where dev=1") + tdSql.checkRows(0) + + tdSql.query("select * from db.st where tag2=1") + tdSql.checkRows(0) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/filterAllUnsignedIntTypes.py b/tests/pytest/query/filterAllUnsignedIntTypes.py new file mode 100644 index 0000000000000000000000000000000000000000..0ff2f2b6eeb1424b0bcfd9a9e54537e8f216f9c6 --- /dev/null +++ b/tests/pytest/query/filterAllUnsignedIntTypes.py @@ -0,0 +1,176 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.powers = [8, 16, 32, 64] + self.types = ["tinyint", "smallint", "int", "bigint"] + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + for i in range(len(self.powers)): + curType = self.types[i] + print("======= Verify filter for %s type =========" % (curType)) + tdLog.debug( + "create table st%s(ts timestamp, num %s unsigned) tags(id %s unsigned)" % + (curType, curType, curType)) + tdSql.execute( + "create table st%s(ts timestamp, num %s unsigned) tags(id %s unsigned)" % + (curType, curType, curType)) + + # create 10 tables, insert 10 rows for each table + for j in range(self.rowNum): + tdSql.execute( + "create table st%s%d using st%s tags(%d)" % + (curType, j + 1, curType, j + 1)) + for k in range(self.rowNum): + tdSql.execute( + "insert into st%s%d values(%d, %d)" % + (curType, j + 1, self.ts + k + 1, j * 10 + k + 1)) + + tdSql.error("insert into st%s10 values(%d, %d)" % + (curType, self.ts + 11, pow(2, self.powers[i]) - 1 )) + tdSql.execute("insert into st%s10 values(%d, %d)" % + (curType, self.ts + 12, pow(2, self.powers[i]) - 2 )) + tdSql.error("insert into st%s10 values(%d, %d)" % + (curType, self.ts + 13, -1 )) + tdSql.execute("insert into st%s10 values(%d, %d)" % + (curType, self.ts + 14, 0 )) + # select all data + # tdSql.query("select * from st%s " % curType) + # tdSql.checkRows(102) + + # > for int type on column + tdSql.query("select * from st%s where num > 50" % curType) + tdSql.checkRows(51) + + # >= for int type on column + tdSql.query("select * from st%s where num >= 50" % curType) + tdSql.checkRows(52) + + # = for int type on column + tdSql.query("select * from st%s where num = 50" % curType) + tdSql.checkRows(1) + + # < for int type on column + tdSql.query("select * from st%s where num < 50" % curType) + tdSql.checkRows(50) + + # <= for int type on column + tdSql.query("select * from st%s where num <= 50" % curType) + tdSql.checkRows(51) + + # <> for int type on column + tdSql.query("select * from st%s where num <> 50" % curType) + tdSql.checkRows(101) + + # != for int type on column + tdSql.query("select * from st%s where num != 50" % curType) + tdSql.checkRows(101) + + # range for int type on column + tdSql.query( + "select * from st%s where num > 50 and num < 100" % + curType) + tdSql.checkRows(49) + + tdSql.query( + "select * from st%s where num >= 50 and num < 100" % + curType) + tdSql.checkRows(50) + + tdSql.query( + "select * from st%s where num > 50 and num <= 100" % + curType) + tdSql.checkRows(50) + + tdSql.query( + "select * from st%s where num >= 50 and num <= 100" % + curType) + tdSql.checkRows(51) + + # > for int type on tag + tdSql.query("select * from st%s where id > 5" % curType) + tdSql.checkRows(52) + + # >= for int type on tag + tdSql.query("select * from st%s where id >= 5" % curType) + tdSql.checkRows(62) + + # = for int type on tag + tdSql.query("select * from st%s where id = 5" % curType) + tdSql.checkRows(10) + + # < for int type on tag + tdSql.query("select * from st%s where id < 5" % curType) + tdSql.checkRows(40) + + # <= for int type on tag + tdSql.query("select * from st%s where id <= 5" % curType) + tdSql.checkRows(50) + + # <> for int type on tag + tdSql.query("select * from st%s where id <> 5" % curType) + tdSql.checkRows(92) + + # != for int type on tag + tdSql.query("select * from st%s where id != 5" % curType) + tdSql.checkRows(92) + + # != for int type on tag + tdSql.query("select * from st%s where id != 5" % curType) + tdSql.checkRows(92) + + # range for int type on tag + tdSql.query("select * from st%s where id > 5 and id < 7" % curType) + tdSql.checkRows(10) + + tdSql.query( + "select * from st%s where id >= 5 and id < 7" % + curType) + tdSql.checkRows(20) + + tdSql.query( + "select * from st%s where id > 5 and id <= 7" % + curType) + tdSql.checkRows(20) + + tdSql.query( + "select * from st%s where id >= 5 and id <= 7" % + curType) + tdSql.checkRows(30) + + print( + "======= Verify filter for %s type finished =========" % + curType) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py index d9e09dae7acdcd8c7401f1370da9f544c850872d..a0e81477096e9c846e109ae71020b40e47b39a84 100644 --- a/tests/pytest/query/last_row_cache.py +++ b/tests/pytest/query/last_row_cache.py @@ -25,18 +25,23 @@ class TDTestCase: self.tables = 10 self.rows = 20 + self.columns = 50 self.perfix = 't' self.ts = 1601481600000 def insertData(self): - print("==============step1") - tdSql.execute("create table st (ts timestamp, c1 int) tags(t1 int)") + print("==============step1") + sql = "create table st(ts timestamp, " + for i in range(self.columns - 1): + sql += "c%d int, " % (i + 1) + sql += "c50 int) tags(t1 int)" + tdSql.execute(sql) for i in range(self.tables): tdSql.execute("create table %s%d using st tags(%d)" % (self.perfix, i, i)) for j in range(self.rows): tc = self.ts + j * 60000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, i, tc, j)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, i, tc, j)) def executeQueries(self): print("==============step2") @@ -66,29 +71,29 @@ class TDTestCase: tdSql.checkData(0, 0, 19) tc = self.ts + 1 * 3600000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, 10)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, 10)) tc = self.ts + 3 * 3600000 - tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) tc = self.ts + 5 * 3600000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, -1)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, -1)) tc = self.ts + 7 * 3600000 - tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) def insertData2(self): tc = self.ts + 1 * 3600000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, 10)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, 10)) tc = self.ts + 3 * 3600000 - tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) tc = self.ts + 5 * 3600000 - tdSql.execute("insert into %s%d values(%d, %d)" %(self.perfix, 1, tc, -1)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, %d)" %(self.perfix, 1, tc, -1)) tc = self.ts + 7 * 3600000 - tdSql.execute("insert into %s%d values(%d, null)" %(self.perfix, 1, tc)) + tdSql.execute("insert into %s%d(ts, c1) values(%d, null)" %(self.perfix, 1, tc)) def executeQueries2(self): # For stable @@ -164,6 +169,9 @@ class TDTestCase: self.executeQueries() self.insertData2() self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() tdSql.execute("alter database test2 cachelast 0") self.executeQueries2() diff --git a/tests/pytest/query/query.py b/tests/pytest/query/query.py index 756aa0eda998aaa0aad30e94b2a7192d7d53bb65..8cec38780e5e873cb07d8ade45ef56b5d4ee43f7 100644 --- a/tests/pytest/query/query.py +++ b/tests/pytest/query/query.py @@ -111,6 +111,18 @@ class TDTestCase: tdSql.query("select * from tb where c5 = 'true' ") tdSql.checkRows(5) + # For jira: https://jira.taosdata.com:18080/browse/TD-2850 + tdSql.execute("create database 'Test' ") + tdSql.execute("use 'Test' ") + tdSql.execute("create table 'TB'(ts timestamp, 'Col1' int) tags('Tag1' int)") + tdSql.execute("insert into 'Tb0' using tb tags(1) values(now, 1)") + tdSql.query("select * from tb") + tdSql.checkRows(1) + + tdSql.query("select * from tb0") + tdSql.checkRows(1) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py new file mode 100644 index 0000000000000000000000000000000000000000..e66a79f5e6d4938bef62eccc93f4cec3de6fe712 --- /dev/null +++ b/tests/pytest/query/query1970YearsAf.py @@ -0,0 +1,258 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import sys +import os +import json +import subprocess +import datetime + + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnode + +class TDTestCase: + + def __init__(self): + self.path = "" + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getcfgPath(self, path): + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug(f"binPath {binPath}") + binPath = os.path.realpath(binPath) + tdLog.debug(f"binPath real path {binPath}") + + if path == "": + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + return self.path + + def getCfgDir(self): + self.getcfgPath(self.path) + self.cfgDir = f"{self.path}/sim/psim/cfg" + return self.cfgDir + + def creatcfg(self): + dbinfo = { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 36500, + "minRows": 100, + "maxRows": 4096, + "comp": 2, + "walLevel": 1, + "cachelast": 0, + "quorum": 1, + "fsync": 3000, + "update": 0 + } + + # set stable schema + stable1 = { + "name": "stb2", + "child_table_exists": "no", + "childtable_count": 10, + "childtable_prefix": "t", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 5000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 1, + "max_sql_len": 65480, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 20000, + "start_timestamp": "1969-12-31 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + + ], + "tags": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + ] + } + + # create different stables like stable1 and add to list super_tables + super_tables = [] + super_tables.append(stable1) + database = { + "dbinfo": dbinfo, + "super_tables": super_tables + } + + cfgdir = self.getCfgDir() + create_table = { + "filetype": "insert", + "cfgdir": cfgdir, + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "/tmp/insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "databases": [database] + } + return create_table + + def createinsertfile(self): + create_table = self.creatcfg() + date = datetime.datetime.now().strftime("%Y%m%d%H%M") + file_create_table = f"/tmp/insert_{date}.json" + + with open(file_create_table, 'w') as f: + json.dump(create_table, f) + return file_create_table + + def inserttable(self, filepath): + create_table_cmd = f"taosdemo -f {filepath} > /dev/null 2>&1" + _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") + + def sqlsquery(self): + # stable query + tdSql.query( + "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(43200) + + tdSql.query( + "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(6800) + + tdSql.query( + "select * from stb2 where stb2.ts > '1969-12-31 23:00:00.000' and stb2.ts <'1970-01-01 01:00:00.000' " + ) + tdSql.checkRows(3590) + + # child-tables query + tdSql.query( + "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(4320) + + tdSql.query( + "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(680) + + tdSql.query( + "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' " + ) + tdSql.checkRows(719) + + tdSql.query( + "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(680) + + tdSql.query( + "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(679) + + tdSql.query( + "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" + ) + tdSql.checkRows(43200) + + # query with timestamp in 'where ...' + tdSql.query( + "select * from stb2 where stb2.ts > -28800000 " + ) + tdSql.checkRows(6790) + + tdSql.query( + "select * from stb2 where stb2.ts > -28800000 and stb2.ts < '1970-01-01 08:00:00.000' " + ) + tdSql.checkRows(6790) + + tdSql.query( + "select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 22:00:00.000' " + ) + tdSql.checkRows(3590) + + def run(self): + s = 'reset query cache' + tdSql.execute(s) + s = 'create database if not exists db' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + + tdLog.info("==========step1:create table stable and child table,then insert data automatically") + insertfile = self.createinsertfile() + self.inserttable(insertfile) + + tdLog.info("==========step2:query join") + self.sqlsquery() + + # after wal and sync, check again + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) + + tdLog.info("==========step3: query join again") + self.sqlsquery() + + # delete temporary file + rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" + _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/queryBetweenAnd.py b/tests/pytest/query/queryBetweenAnd.py new file mode 100644 index 0000000000000000000000000000000000000000..cd4320f5235222049f3c1dfc6a7bb8a1ab901b61 --- /dev/null +++ b/tests/pytest/query/queryBetweenAnd.py @@ -0,0 +1,206 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + tdSql.execute( + '''create table if not exists supt + (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint) + tags(location binary(64), type int, isused bool , family nchar(64))''' + ) + tdSql.execute("create table t1 using supt tags('beijing', 1, 1, '自行车')") + tdSql.execute("create table t2 using supt tags('shanghai', 2, 0, '拖拉机')") + + tdLog.printNoPrefix("==========step2:insert data") + for i in range(10): + tdSql.execute( + f"insert into t1 values (now+{i}m, {32767+i}, {20.0+i/10}, {2**31+i}, {3.4*10**38+i/10}, {127+i}, {i})" + ) + tdSql.execute( + f"insert into t2 values (now-{i}m, {-32767-i}, {20.0-i/10}, {-i-2**31}, {-i/10-3.4*10**38}, {-127-i}, {-i})" + ) + tdSql.execute( + f"insert into t1 values (now+11m, {2**31-1}, {pow(10,37)*34}, {pow(2,63)-1}, {1.7*10**308}, 32767, 127)" + ) + tdSql.execute( + f"insert into t2 values (now-11m, {1-2**31}, {-3.4*10**38}, {1-2**63}, {-1.7*10**308}, -32767, -127)" + ) + tdSql.execute( + f"insert into t2 values (now-12m, null , {-3.4*10**38}, null , {-1.7*10**308}, null , null)" + ) + + tdLog.printNoPrefix("==========step3:query timestamp type") + + tdSql.query("select * from t1 where ts between now-1m and now+10m") + tdSql.checkRows(10) + tdSql.query("select * from t1 where ts between '2021-01-01 00:00:00.000' and '2121-01-01 00:00:00.000'") + tdSql.checkRows(11) + tdSql.query("select * from t1 where ts between '1969-01-01 00:00:00.000' and '1969-12-31 23:59:59.999'") + tdSql.checkRows(0) + tdSql.query("select * from t1 where ts between -2793600 and 31507199") + tdSql.checkRows(0) + tdSql.query("select * from t1 where ts between 1609430400000 and 4765104000000") + tdSql.checkRows(11) + + tdLog.printNoPrefix("==========step4:query int type") + + tdSql.query("select * from t1 where c1 between 32767 and 32776") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c1 between 32766.9 and 32776.1") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c1 between 32776 and 32767") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c1 between 'a' and 'e'") + # tdSql.query("select * from t1 where c1 between 0x64 and 0x69") + # tdSql.checkRows(6) + tdSql.error("select * from t1 where c1 not between 100 and 106") + tdSql.query(f"select * from t1 where c1 between {2**31-2} and {2**31+1}") + tdSql.checkRows(1) + tdSql.error(f"select * from t2 where c1 between null and {1-2**31}") + # tdSql.checkRows(3) + tdSql.query(f"select * from t2 where c1 between {-2**31} and {1-2**31}") + tdSql.checkRows(1) + + tdLog.printNoPrefix("==========step5:query float type") + + tdSql.query("select * from t1 where c2 between 20.0 and 21.0") + tdSql.checkRows(10) + tdSql.query(f"select * from t1 where c2 between {-3.4*10**38-1} and {3.4*10**38+1}") + tdSql.checkRows(11) + tdSql.query("select * from t1 where c2 between 21.0 and 20.0") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c2 between 'DC3' and 'SYN'") + tdSql.error("select * from t1 where c2 not between 0.1 and 0.2") + # tdSql.query(f"select * from t1 where c2 between {pow(10,38)*3.4} and {pow(10,38)*3.4+1}") + # tdSql.checkRows(1) + tdSql.query(f"select * from t2 where c2 between {-3.4*10**38-1} and {-3.4*10**38}") + tdSql.checkRows(0) + tdSql.error(f"select * from t2 where c2 between null and {-3.4*10**38}") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step6:query bigint type") + + tdSql.query(f"select * from t1 where c3 between {2**31} and {2**31+10}") + tdSql.checkRows(10) + tdSql.error(f"select * from t1 where c3 between {-2**63} and {2**63}") + # tdSql.checkRows(11) + tdSql.query(f"select * from t1 where c3 between {2**31+10} and {2**31}") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c3 between 'a' and 'z'") + tdSql.error("select * from t1 where c3 not between 1 and 2") + tdSql.query(f"select * from t1 where c3 between {2**63-2} and {2**63-1}") + tdSql.checkRows(1) + tdSql.error(f"select * from t2 where c3 between {-2**63} and {1-2**63}") + # tdSql.checkRows(3) + tdSql.error(f"select * from t2 where c3 between null and {1-2**63}") + # tdSql.checkRows(2) + + tdLog.printNoPrefix("==========step7:query double type") + + tdSql.query(f"select * from t1 where c4 between {3.4*10**38} and {3.4*10**38+10}") + tdSql.checkRows(10) + tdSql.query(f"select * from t1 where c4 between {1.7*10**308+1} and {1.7*10**308+2}") + # 因为精度原因,在超出bigint边界后,数值不能进行准确的判断 + # tdSql.checkRows(0) + tdSql.query(f"select * from t1 where c4 between {3.4*10**38+10} and {3.4*10**38}") + # tdSql.checkRows(0) + tdSql.error("select * from t1 where c4 between 'a' and 'z'") + tdSql.error("select * from t1 where c4 not between 1 and 2") + tdSql.query(f"select * from t1 where c4 between {1.7*10**308} and {1.7*10**308+1}") + tdSql.checkRows(1) + tdSql.query(f"select * from t2 where c4 between {-1.7*10**308-1} and {-1.7*10**308}") + # tdSql.checkRows(3) + tdSql.error(f"select * from t2 where c4 between null and {-1.7*10**308}") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step8:query smallint type") + + tdSql.query("select * from t1 where c5 between 127 and 136") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c5 between 126.9 and 135.9") + tdSql.checkRows(9) + tdSql.query("select * from t1 where c5 between 136 and 127") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c5 between '~' and 'ˆ'") + tdSql.error("select * from t1 where c5 not between 1 and 2") + tdSql.query("select * from t1 where c5 between 32767 and 32768") + tdSql.checkRows(1) + tdSql.query("select * from t2 where c5 between -32768 and -32767") + tdSql.checkRows(1) + tdSql.error("select * from t2 where c5 between null and -32767") + # tdSql.checkRows(1) + + tdLog.printNoPrefix("==========step9:query tinyint type") + + tdSql.query("select * from t1 where c6 between 0 and 9") + tdSql.checkRows(10) + tdSql.query("select * from t1 where c6 between -1.1 and 8.9") + tdSql.checkRows(9) + tdSql.query("select * from t1 where c6 between 9 and 0") + tdSql.checkRows(0) + tdSql.error("select * from t1 where c6 between 'NUL' and 'HT'") + tdSql.error("select * from t1 where c6 not between 1 and 2") + tdSql.query("select * from t1 where c6 between 127 and 128") + tdSql.checkRows(1) + tdSql.query("select * from t2 where c6 between -128 and -127") + tdSql.checkRows(1) + tdSql.error("select * from t2 where c6 between null and -127") + # tdSql.checkRows(3) + + tdLog.printNoPrefix("==========step10:invalid query type") + + tdSql.query("select * from supt where location between 'beijing' and 'shanghai'") + tdSql.checkRows(23) + # 非0值均解析为1,因此"between 负值 and o"解析为"between 1 and 0" + tdSql.query("select * from supt where isused between 0 and 1") + tdSql.checkRows(23) + tdSql.query("select * from supt where isused between -1 and 0") + tdSql.checkRows(0) + tdSql.error("select * from supt where isused between false and true") + tdSql.query("select * from supt where family between '拖拉机' and '自行车'") + tdSql.checkRows(23) + + tdLog.printNoPrefix("==========step11:query HEX/OCT/BIN type") + + tdSql.error("select * from t1 where c6 between 0x7f and 0x80") # check filter HEX + tdSql.error("select * from t1 where c6 between 0b1 and 0b11111") # check filter BIN + tdSql.error("select * from t1 where c6 between 0b1 and 0x80") + tdSql.error("select * from t1 where c6=0b1") + tdSql.error("select * from t1 where c6=0x1") + # 八进制数据会按照十进制数据进行判定 + tdSql.query("select * from t1 where c6 between 01 and 0200") # check filter OCT + tdSql.checkRows(10) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/queryFillTest.py b/tests/pytest/query/queryFillTest.py index 9fd898041a525f9aaba65bda1233f94b1bb922e6..e50d02faf27f33191f4a76c535443f6802e0ed28 100644 --- a/tests/pytest/query/queryFillTest.py +++ b/tests/pytest/query/queryFillTest.py @@ -47,11 +47,38 @@ class TDTestCase: tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h)") tdSql.checkRows(139) + tdSql.checkData(0, 1, -1.5) + tdSql.checkData(138, 1, -1.0) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(none)") + tdSql.checkRows(139) + tdSql.checkData(0, 1, -1.5) + tdSql.checkData(138, 1, -1.0) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(value, 2.0)") + tdSql.checkRows(141) + tdSql.checkData(0, 1, 2.0) + tdSql.checkData(140, 1, 2.0) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(prev)") + tdSql.checkRows(141) + tdSql.checkData(0, 1, None) + tdSql.checkData(140, 1, -1.0) tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(null)") tdSql.checkRows(141) tdSql.checkData(0, 1, None) - tdSql.checkData(140, 1, None) + tdSql.checkData(140, 1, None) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(linear)") + tdSql.checkRows(141) + tdSql.checkData(0, 1, None) + tdSql.checkData(140, 1, None) + + tdSql.query("select first(col1) - avg(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' interval(1h) fill(next)") + tdSql.checkRows(141) + tdSql.checkData(0, 1, -1.5) + tdSql.checkData(140, 1, None) tdSql.query("select max(col1) - min(col1) from stb where ts > '2018-09-17 08:00:00.000' and ts < '2018-09-23 04:36:40.000' and id = 1 group by loc, id") rows = tdSql.queryRows diff --git a/tests/pytest/query/queryGroupbySort.py b/tests/pytest/query/queryGroupbySort.py index 28ef897f42924d3e41c6598cac6b4c51e33d38ef..063db936087c125e45051da0094b57a9fd184b9b 100644 --- a/tests/pytest/query/queryGroupbySort.py +++ b/tests/pytest/query/queryGroupbySort.py @@ -28,19 +28,35 @@ class TDTestCase: def run(self): tdSql.prepare() - tdSql.execute( - "create table stb(ts timestamp,i int) tags (p_id nchar(20));") - tdSql.execute( - "insert into tb using stb tags('11231') values (%d, %d) (%d, %d) (%d, %d) (%d, %d)" - % (self.ts, 12, self.ts + 1, 15, self.ts + 2, 15, self.ts + 3, 12)) + tdSql.execute("CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int, t3 float, t4 double)") + tdSql.execute("CREATE TABLE D1001 USING meters TAGS ('Beijing.Chaoyang', 2 , NULL, NULL)") + tdSql.execute("CREATE TABLE D1002 USING meters TAGS ('Beijing.Chaoyang', 3 , NULL , 1.7)") + tdSql.execute("CREATE TABLE D1003 USING meters TAGS ('Beijing.Chaoyang', 3 , 1.1 , 1.7)") + tdSql.execute("INSERT INTO D1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) (1538548696800, 12.3, 221, 0.31)") + tdSql.execute("INSERT INTO D1002 VALUES (1538548685001, 10.5, 220, 0.28) (1538548696800, 12.3, 221, 0.31)") + tdSql.execute("INSERT INTO D1003 VALUES (1538548685001, 10.5, 220, 0.28) (1538548696800, 12.3, 221, 0.31)") + tdSql.query("SELECT SUM(current), AVG(voltage) FROM meters WHERE groupId > 1 INTERVAL(1s) GROUP BY location order by ts DESC") + tdSql.checkRows(3) + tdSql.checkData(0, 0, "2018-10-03 14:38:16") + tdSql.checkData(1, 0, "2018-10-03 14:38:15") + tdSql.checkData(2, 0, "2018-10-03 14:38:05") - tdSql.query(''' select last(ts) p_time,i from stb where p_id='11231' and ts>=%d and ts <=%d - group by i order by time desc limit 100 ''' % (self.ts, self.ts + 4)) - tdSql.checkRows(2) - tdSql.checkData(0, 0, "2018-09-17 09:00:00.003000") - tdSql.checkData(1, 0, "2018-09-17 09:00:00.002000") + tdSql.query("SELECT SUM(current), AVG(voltage) FROM meters WHERE groupId > 1 INTERVAL(1s) GROUP BY location order by ts ASC") + tdSql.checkRows(3) + tdSql.checkData(0, 0, "2018-10-03 14:38:05") + tdSql.checkData(1, 0, "2018-10-03 14:38:15") + tdSql.checkData(2, 0, "2018-10-03 14:38:16") + tdSql.error("SELECT SUM(current) as s, AVG(voltage) FROM meters WHERE groupId > 1 INTERVAL(1s) GROUP BY location order by s ASC") + tdSql.error("SELECT SUM(current) as s, AVG(voltage) FROM meters WHERE groupId > 1 INTERVAL(1s) GROUP BY location order by s DESC") + + #add for TD-3170 + tdSql.query("select avg(current) from meters group by t3;") + tdSql.checkData(0, 0, 11.6) + tdSql.query("select avg(current) from meters group by t4;") + tdSql.query("select avg(current) from meters group by t3,t4;") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/query/queryNormal.py b/tests/pytest/query/queryNormal.py index 13393117d642ec1fbdf839d59429428ffee79a27..4e3f9dd3bdd6376ac1069c4e084944a577ce4a70 100644 --- a/tests/pytest/query/queryNormal.py +++ b/tests/pytest/query/queryNormal.py @@ -94,8 +94,9 @@ class TDTestCase: tdSql.query("select * from stb1 limit 2 offset 3") tdSql.checkRows(1) - # query ... alias for table ---- bug - tdSql.error("select t.ts from tb1 t") + # query ... alias for table + tdSql.query("select t.ts from tb1 t") + tdSql.checkRows(2) # query ... tbname tdSql.query("select tbname from stb1") diff --git a/tests/pytest/query/queryNullValueTest.py b/tests/pytest/query/queryNullValueTest.py index 9920543b3a8d78363197a632eabe47ac6f82fa2e..5a95b369fe39d6bc1216847b3f108dff7d7171b7 100644 --- a/tests/pytest/query/queryNullValueTest.py +++ b/tests/pytest/query/queryNullValueTest.py @@ -25,7 +25,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.types = ["tinyint", "smallint", "int", "bigint", "float", "double", "bool", "binary(10)", "nchar(10)"] + self.types = ["tinyint", "smallint", "int", "bigint", "float", "double", "bool", "binary(10)", "nchar(10)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"] self.ts = 1537146000000 def checkNullValue(self, result): diff --git a/tests/pytest/query/querySort.py b/tests/pytest/query/querySort.py index 649e0dc1cb3191ba08b3f2da0a5edee3afc66575..17022bdc41057bcb67e1530a2cb6d399bada20ff 100644 --- a/tests/pytest/query/querySort.py +++ b/tests/pytest/query/querySort.py @@ -73,19 +73,19 @@ class TDTestCase: print("======= step 1: create table and insert data =========") tdLog.debug( ''' create table st(ts timestamp, tbcol1 tinyint, tbcol2 smallint, tbcol3 int, tbcol4 bigint, tbcol5 float, tbcol6 double, - tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20)) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, - tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20))''') + tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20), tbcol11 tinyint unsigned, tbcol12 smallint unsigned, tbcol13 int unsigned, tbcol14 bigint unsigned) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, + tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20), tagcol11 tinyint unsigned, tagcol12 smallint unsigned, tagcol13 int unsigned, tagcol14 bigint unsigned)''') tdSql.execute( ''' create table st(ts timestamp, tbcol1 tinyint, tbcol2 smallint, tbcol3 int, tbcol4 bigint, tbcol5 float, tbcol6 double, - tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20)) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, - tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20))''') + tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20), tbcol11 tinyint unsigned, tbcol12 smallint unsigned, tbcol13 int unsigned, tbcol14 bigint unsigned) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, + tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20), tagcol11 tinyint unsigned, tagcol12 smallint unsigned, tagcol13 int unsigned, tagcol14 bigint unsigned)''') for i in range(self.rowNum): - tdSql.execute("create table st%d using st tags(%d, %d, %d, %d, %f, %f, %d, 'tag%d', '标签%d')" % ( - i + 1, i + 1, i + 1, i + 1, i + 1, 1.1 * (i + 1), 1.23 * (i + 1), (i + 1) % 2, i + 1, i + 1)) + tdSql.execute("create table st%d using st tags(%d, %d, %d, %d, %f, %f, %d, 'tag%d', '标签%d', %d, %d, %d, %d)" % ( + i + 1, i + 1, i + 1, i + 1, i + 1, 1.1 * (i + 1), 1.23 * (i + 1), (i + 1) % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) for j in range(self.rowNum): - tdSql.execute("insert into st%d values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" % ( - i + 1, self.ts + 10 * (i + 1) + j + 1, j + 1, j + 1, j + 1, j + 1, 1.1 * (j + 1), 1.23 * (j + 1), (j + 1) % 2, j + 1, j + 1)) + tdSql.execute("insert into st%d values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % ( + i + 1, self.ts + 10 * (i + 1) + j + 1, j + 1, j + 1, j + 1, j + 1, 1.1 * (j + 1), 1.23 * (j + 1), (j + 1) % 2, j + 1, j + 1, i + 1, i + 1, i + 1, i + 1)) print("======= step 2: verify order for each column =========") # sort for timestamp in asc order diff --git a/tests/pytest/query/sliding.py b/tests/pytest/query/sliding.py index 810d90117a73bba81ea010f81aa605196a496dc4..d7a7c9a36964796dd530f913523f8f4d344d467a 100644 --- a/tests/pytest/query/sliding.py +++ b/tests/pytest/query/sliding.py @@ -52,7 +52,15 @@ class TDTestCase: tdSql.query("select count(*) from meters group by loc") tdSql.checkRows(2) - tdSql.error("select * from meters group by loc sliding(5s)") + tdSql.error("select * from meters group by loc sliding(5s)") + + # Fix defect: https://jira.taosdata.com:18080/browse/TD-2700 + tdSql.execute("create database test") + tdSql.execute("use test") + tdSql.execute("create table t1(ts timestamp, k int)") + tdSql.execute("insert into t1 values(1500000001000, 0)") + tdSql.query("select sum(k) from t1 interval(1d) sliding(1h)") + tdSql.checkRows(24) def stop(self): tdSql.close() diff --git a/tests/pytest/stable/insert.py b/tests/pytest/stable/insert.py index 3d37e6726c965a8963f728fdd1d8dc89e1cee339..0ef816da8d02fc6dcb48954435aad51342248e56 100644 --- a/tests/pytest/stable/insert.py +++ b/tests/pytest/stable/insert.py @@ -47,6 +47,40 @@ class TDTestCase: tdSql.query("select * from db.st where dev='dev_02'") tdSql.checkRows(1) + #For: https://jira.taosdata.com:18080/browse/TD-2671 + print("==============step3") + tdSql.execute( + "create stable if not exists stb (ts timestamp, tagtype int) tags(dev nchar(50))") + tdSql.execute( + 'CREATE TABLE if not exists dev_01 using stb tags("dev_01")') + tdSql.execute( + 'CREATE TABLE if not exists dev_02 using stb tags("dev_02")') + + print("==============step4") + + tdSql.execute( + """INSERT INTO dev_01(ts, tagtype) VALUES('2020-05-13 10:00:00.000', 1), + ('2020-05-13 10:00:00.001', 1) + dev_02 VALUES('2020-05-13 10:00:00.001', 1)""") + + tdSql.query("select * from db.stb where dev='dev_01'") + tdSql.checkRows(2) + + tdSql.query("select * from db.stb where dev='dev_02'") + tdSql.checkRows(1) + + tdSql.query("describe db.stb") + tdSql.checkRows(3) + + tdSql.execute("alter stable db.stb add tag t1 int") + tdSql.query("describe db.stb") + tdSql.checkRows(4) + + tdSql.execute("drop stable db.stb") + tdSql.query("show stables") + tdSql.checkRows(1) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/stream/stream3.py b/tests/pytest/stream/stream3.py new file mode 100644 index 0000000000000000000000000000000000000000..9a5c6c9aeca08bff1c94861255919255eef89100 --- /dev/null +++ b/tests/pytest/stream/stream3.py @@ -0,0 +1,108 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + ts = 1500000000000 + tbNum = 10 + rowNum = 20 + + tdSql.prepare() + + tdLog.info("===== step1 =====") + tdSql.execute( + "create table stb0(ts timestamp, col1 binary(20), col2 nchar(20)) tags(tgcol int)") + for i in range(tbNum): + tdSql.execute("create table tb%d using stb0 tags(%d)" % (i, i)) + for j in range(rowNum): + tdSql.execute( + "insert into tb%d values (%d, 'binary%d', 'nchar%d')" % + (i, ts + 60000 * j, j, j)) + tdSql.execute("insert into tb0 values(%d, null, null)" % (ts + 10000000)) + time.sleep(0.1) + + tdLog.info("===== step2 =====") + tdSql.query( + "select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + + tdSql.query("show tables") + tdSql.checkRows(tbNum) + tdSql.execute( + "create table s0 as select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step3 =====") + tdSql.waitedQuery("select * from s0", 1, 120) + try: + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + except Exception as e: + tdLog.info(repr(e)) + + tdLog.info("===== step4 =====") + tdSql.execute("drop table s0") + tdSql.query("show tables") + tdSql.checkRows(tbNum) + + tdLog.info("===== step5 =====") + tdSql.error("select * from s0") + + tdLog.info("===== step6 =====") + time.sleep(0.1) + tdSql.execute( + "create table s0 as select count(*), count(col1), count(col2) from tb0 interval(1d)") + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + tdLog.info("===== step7 =====") + tdSql.waitedQuery("select * from s0", 1, 120) + try: + tdSql.checkData(0, 1, rowNum + 1) + tdSql.checkData(0, 2, rowNum) + tdSql.checkData(0, 3, rowNum) + except Exception as e: + tdLog.info(repr(e)) + + tdLog.info("===== step8 =====") + tdSql.query( + "select count(*), count(col1), count(col2) from stb0 interval(1d)") + tdSql.checkData(0, 1, rowNum * tbNum + 1) + tdSql.checkData(0, 2, rowNum * tbNum) + tdSql.checkData(0, 3, rowNum * tbNum) + tdSql.query("show tables") + tdSql.checkRows(tbNum + 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/add.py b/tests/pytest/tag_lite/add.py index 49e2f8ab6990c24e90e9d1b01df957bc0d3107cd..eb52e9cb9b8f595d77e4d816c2fb2c1317501cfb 100644 --- a/tests/pytest/tag_lite/add.py +++ b/tests/pytest/tag_lite/add.py @@ -141,6 +141,98 @@ class TDTestCase: # TSIM: sql select * from $mt where tgcol2 = 1 -x step2 tdLog.info('select * from %s where tgcol2 = 1 -x step2' % (mt)) tdSql.error('select * from %s where tgcol2 = 1' % (mt)) + tdLog.info('=============== step2-1') + # TSIM: $i = 2 + i = 21 + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 + # bool, tgcol2 int) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 bool, tgcol2 int unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 bool, tgcol2 int unsigned)' % + (mt)) + # TSIM: sql create table $tb using $mt tags( 1, 2 ) + tdLog.info('create table %s using %s tags( 1, 2 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1, 2 )' % (tb, mt)) + # TSIM: sql insert into $tb values(now, 1) + tdLog.info('insert into %s values(now, 1)' % (tb)) + tdSql.execute('insert into %s values(now, 1)' % (tb)) + # TSIM: sql select * from $mt where tgcol2 = 2 + tdLog.info('select * from %s where tgcol2 = 2' % (mt)) + tdSql.query('select * from %s where tgcol2 = 2' % (mt)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 2 then + tdLog.info('tdSql.checkData(0, 3, 2)') + tdSql.checkData(0, 3, 2) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql alter table $mt drop tag tgcol2 + tdLog.info('alter table %s drop tag tgcol2' % (mt)) + tdSql.execute('alter table %s drop tag tgcol2' % (mt)) + # TSIM: sql alter table $mt add tag tgcol4 int + tdLog.info('alter table %s add tag tgcol4 int unsigned' % (mt)) + tdSql.execute('alter table %s add tag tgcol4 int unsigned' % (mt)) + tdLog.info('select * from %s where tgcol4=6' % (mt)) + tdSql.query('select * from %s where tgcol4=6' % (mt)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: sql alter table $tb set tag tgcol4 =4 + tdLog.info('alter table %s set tag tgcol4 =4' % (tb)) + tdSql.execute('alter table %s set tag tgcol4 =4' % (tb)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: + # TSIM: sql select * from $mt where tgcol4 = 4 + tdLog.info('select * from %s where tgcol4 = 4' % (mt)) + tdSql.query('select * from %s where tgcol4 = 4' % (mt)) + # TSIM: print $data01 $data02 $data03 + tdLog.info('$data01 $data02 $data03') + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 4 then + tdLog.info('tdSql.checkData(0, 3, 4)') + tdSql.checkData(0, 3, 4) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where tgcol2 = 1 -x step2 + tdLog.info('select * from %s where tgcol2 = 1 -x step2' % (mt)) + tdSql.error('select * from %s where tgcol2 = 1' % (mt)) # TSIM: return -1 # TSIM: step2: # TSIM: @@ -235,6 +327,96 @@ class TDTestCase: # TSIM: sql select * from $mt where tgcol2 = 1 -x step3 tdLog.info('select * from %s where tgcol2 = 1 -x step3' % (mt)) tdSql.error('select * from %s where tgcol2 = 1' % (mt)) + tdLog.info('=============== step3-1') + # TSIM: $i = 3 + i = 31 + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 + # smallint, tgcol2 tinyint) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 smallint unsigned, tgcol2 tinyint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 smallint unsigned, tgcol2 tinyint unsigned)' % + (mt)) + # TSIM: sql create table $tb using $mt tags( 1, 2 ) + tdLog.info('create table %s using %s tags( 1, 2 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1, 2 )' % (tb, mt)) + # TSIM: sql insert into $tb values(now, 1) + tdLog.info('insert into %s values(now, 1)' % (tb)) + tdSql.execute('insert into %s values(now, 1)' % (tb)) + # TSIM: sql select * from $mt where tgcol2 = 2 + tdLog.info('select * from %s where tgcol2 = 2' % (mt)) + tdSql.query('select * from %s where tgcol2 = 2' % (mt)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 2 then + tdLog.info('tdSql.checkData(0, 3, 2)') + tdSql.checkData(0, 3, 2) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql alter table $mt drop tag tgcol2 + tdLog.info('alter table %s drop tag tgcol2' % (mt)) + tdSql.execute('alter table %s drop tag tgcol2' % (mt)) + # TSIM: sql alter table $mt add tag tgcol4 tinyint + tdLog.info('alter table %s add tag tgcol4 tinyint unsigned' % (mt)) + tdSql.execute('alter table %s add tag tgcol4 tinyint unsigned' % (mt)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: sql alter table $tb set tag tgcol4=4 + tdLog.info('alter table %s set tag tgcol4=4' % (tb)) + tdSql.execute('alter table %s set tag tgcol4=4' % (tb)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: + # TSIM: sql select * from $mt where tgcol4 = 4 + tdLog.info('select * from %s where tgcol4 = 4' % (mt)) + tdSql.query('select * from %s where tgcol4 = 4' % (mt)) + # TSIM: print $data01 $data02 $data03 + tdLog.info('$data01 $data02 $data03') + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 4 then + tdLog.info('tdSql.checkData(0, 3, 4)') + tdSql.checkData(0, 3, 4) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where tgcol2 = 1 -x step3 + tdLog.info('select * from %s where tgcol2 = 1 -x step3' % (mt)) + tdSql.error('select * from %s where tgcol2 = 1' % (mt)) # TSIM: return -1 # TSIM: step3: # TSIM: @@ -353,6 +535,120 @@ class TDTestCase: # TSIM: sql select * from $mt where tgcol2 = 1 -x step4 tdLog.info('select * from %s where tgcol2 = 1 -x step4' % (mt)) tdSql.error('select * from %s where tgcol2 = 1' % (mt)) + tdLog.info('=============== step4-1') + # TSIM: $i = 4 + i = 41 + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 + # bigint, tgcol2 float) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 bigint unsigned, tgcol2 float)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 bigint unsigned, tgcol2 float)' % + (mt)) + # TSIM: sql create table $tb using $mt tags( 1, 2 ) + tdLog.info('create table %s using %s tags( 1, 2 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1, 2 )' % (tb, mt)) + # TSIM: sql insert into $tb values(now, 1) + tdLog.info('insert into %s values(now, 1)' % (tb)) + tdSql.execute('insert into %s values(now, 1)' % (tb)) + # TSIM: sql select * from $mt where tgcol2 = 2 + tdLog.info('select * from %s where tgcol2 = 2' % (mt)) + tdSql.query('select * from %s where tgcol2 = 2' % (mt)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 2.00000 then + tdLog.info('tdSql.checkData(0, 3, 2.00000)') + tdSql.checkData(0, 3, 2.00000) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql describe $tb + tdLog.info('describe %s' % (tb)) + tdSql.query('describe %s' % (tb)) + # TSIM: if $data21 != BIGINT then + tdLog.info('tdSql.checkDataType(2, 1, "BIGINT UNSIGNED")') + tdSql.checkDataType(2, 1, "BIGINT") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data31 != FLOAT then + tdLog.info('tdSql.checkDataType(3, 1, "FLOAT")') + tdSql.checkDataType(3, 1, "FLOAT") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data23 != 1 then + tdLog.info('tdSql.checkData(2, 3, TAG)') + tdSql.checkData(2, 3, "TAG") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data33 != 2.000000 then + tdLog.info('tdSql.checkData(3, 3, 2.000000)') + tdSql.checkData(3, 3, "TAG") + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql alter table $mt drop tag tgcol2 + tdLog.info('alter table %s drop tag tgcol2' % (mt)) + tdSql.execute('alter table %s drop tag tgcol2' % (mt)) + # TSIM: sql alter table $mt add tag tgcol4 float + tdLog.info('alter table %s add tag tgcol4 float' % (mt)) + tdSql.execute('alter table %s add tag tgcol4 float' % (mt)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: sql alter table $tb set tag tgcol4=4 + tdLog.info('alter table %s set tag tgcol4=4' % (tb)) + tdSql.execute('alter table %s set tag tgcol4=4' % (tb)) + # TSIM: sql reset query cache + tdLog.info('reset query cache') + tdSql.execute('reset query cache') + # TSIM: + # TSIM: sql select * from $mt where tgcol4 = 4 + tdLog.info('select * from %s where tgcol4 = 4' % (mt)) + tdSql.query('select * from %s where tgcol4 = 4' % (mt)) + # TSIM: print $data01 $data02 $data03 + tdLog.info('$data01 $data02 $data03') + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, 1)') + tdSql.checkData(0, 2, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 4.00000 then + tdLog.info('tdSql.checkData(0, 3, 4.00000)') + tdSql.checkData(0, 3, 4.00000) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where tgcol2 = 1 -x step4 + tdLog.info('select * from %s where tgcol2 = 1 -x step4' % (mt)) + tdSql.error('select * from %s where tgcol2 = 1' % (mt)) # TSIM: return -1 # TSIM: step4: # TSIM: @@ -408,12 +704,18 @@ class TDTestCase: # TSIM: sql alter table $mt add tag tgcol4 smallint tdLog.info('alter table %s add tag tgcol4 smallint' % (mt)) tdSql.execute('alter table %s add tag tgcol4 smallint' % (mt)) + # TSIM: sql alter table $mt add tag tgcol5 smallint unsigned + tdLog.info('alter table %s add tag tgcol5 smallint unsigned' % (mt)) + tdSql.execute('alter table %s add tag tgcol5 smallint unsigned' % (mt)) # TSIM: sql reset query cache tdLog.info('reset query cache') tdSql.execute('reset query cache') # TSIM: sql alter table $tb set tag tgcol4=4 tdLog.info('alter table %s set tag tgcol4=4' % (tb)) tdSql.execute('alter table %s set tag tgcol4=4' % (tb)) + # TSIM: sql alter table $tb set tag tgcol5=5 + tdLog.info('alter table %s set tag tgcol5=5' % (tb)) + tdSql.execute('alter table %s set tag tgcol5=5' % (tb)) # TSIM: sql reset query cache tdLog.info('reset query cache') tdSql.execute('reset query cache') @@ -441,6 +743,11 @@ class TDTestCase: # TSIM: if $data03 != 4 then tdLog.info('tdSql.checkData(0, 3, 4)') tdSql.checkData(0, 3, 4) + tdSql.query('select * from %s where tgcol5 = 5' % (mt)) + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + tdSql.checkData(0, 2, 1.000000000) + tdSql.checkData(0, 4, 5) # TSIM: return -1 # TSIM: endi # TSIM: diff --git a/tests/pytest/tag_lite/alter_tag.py b/tests/pytest/tag_lite/alter_tag.py new file mode 100644 index 0000000000000000000000000000000000000000..9e5abb6c134840ecb4ab52c7d3a6ab623885e12b --- /dev/null +++ b/tests/pytest/tag_lite/alter_tag.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "CREATE TABLE IF NOT EXISTS ampere (ts TIMESTAMP(8),ampere DOUBLE(8)) TAGS (device_name BINARY(50),build_id BINARY(50),project_id BINARY(50),alias BINARY(50))") + tdSql.execute("insert into d1001 using ampere tags('test', '2', '2', '2') VALUES (now, 123)") + tdSql.execute("ALTER TABLE ampere ADD TAG variable_id BINARY(50)") + + print("==============step2") + + tdSql.execute("insert into d1002 using ampere tags('test', '2', '2', '2', 'test') VALUES (now, 124)") + + tdSql.query("select * from ampere") + tdSql.checkRows(2) + tdSql.checkData(0, 6, None) + tdSql.checkData(1, 6, 'test') + + # Test case for: https://jira.taosdata.com:18080/browse/TD-2423 + tdSql.execute("create table stb(ts timestamp, col1 int, col2 nchar(20)) tags(tg1 int, tg2 binary(20), tg3 nchar(25))") + tdSql.execute("insert into tb1 using stb(tg1, tg3) tags(1, 'test1') values(now, 1, 'test1')") + tdSql.query("select *, tg1, tg2, tg3 from tb1") + tdSql.checkRows(1) + tdSql.checkData(0, 3, 1) + tdSql.checkData(0, 4, None) + tdSql.checkData(0, 5, 'test1') + + tdSql.execute("create table tb2 using stb(tg3, tg2) tags('test3', 'test2')") + tdSql.query("select tg1, tg2, tg3 from tb2") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, 'test2') + tdSql.checkData(0, 2, 'test3') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/delete.py b/tests/pytest/tag_lite/delete.py index 17b4deb815e7a05aaa5eaf33685b95e144d3d59b..5e82a96bf41608782fca09d6aeb8b18aa1a727ef 100644 --- a/tests/pytest/tag_lite/delete.py +++ b/tests/pytest/tag_lite/delete.py @@ -869,6 +869,91 @@ class TDTestCase: # TSIM: sql alter table $mt drop tag tgcol6 tdLog.info('alter table %s drop tag tgcol6' % (mt)) tdSql.execute('alter table %s drop tag tgcol6' % (mt)) + tdLog.info('=============== step14') + # TSIM: $i = 14 + i = 14 + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 + # binary(10), tgcol2 int, tgcol3 smallint, tgcol4 binary(11), tgcol5 + # double, tgcol6 binary(20)) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 binary(10), tgcol2 int unsigned, tgcol3 smallint unsigned, tgcol4 binary(11), tgcol5 double, tgcol6 binary(20), tgcol7 tinyint unsigned, tgcol8 bigint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol1 binary(10), tgcol2 int unsigned, tgcol3 smallint unsigned, tgcol4 binary(11), tgcol5 double, tgcol6 binary(20), tgcol7 tinyint unsigned, tgcol8 bigint unsigned)' % + (mt)) + # TSIM: sql create table $tb using $mt tags( '1', 2, 3, '4', 5, '6' ) + tdLog.info( + 'create table %s using %s tags( "1", 2, 3, "4", 5, "6", 7, 8 )' % + (tb, mt)) + tdSql.execute( + 'create table %s using %s tags( "1", 2, 3, "4", 5, "6", 7, 8 )' % + (tb, mt)) + # TSIM: sql insert into $tb values(now, 1) + tdLog.info('insert into %s values(now, 1)' % (tb)) + tdSql.execute('insert into %s values(now, 1)' % (tb)) + # TSIM: sql select * from $mt where tgcol1 = '1' + tdLog.info('select * from %s where tgcol1 = "1"' % (mt)) + tdSql.query('select * from %s where tgcol1 = "1"' % (mt)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data01 != 1 then + tdLog.info('tdSql.checkData(0, 1, 1)') + tdSql.checkData(0, 1, 1) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data02 != 1 then + tdLog.info('tdSql.checkData(0, 2, "1")') + tdSql.checkData(0, 2, "1") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data03 != 2 then + tdLog.info('tdSql.checkData(0, 3, 2)') + tdSql.checkData(0, 3, 2) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data04 != 3 then + tdLog.info('tdSql.checkData(0, 4, 3)') + tdSql.checkData(0, 4, 3) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data05 != 4 then + tdLog.info('tdSql.checkData(0, 5, "4")') + tdSql.checkData(0, 5, "4") + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data06 != 5.000000000 then + tdLog.info('tdSql.checkData(0, 6, 5.000000000)') + tdSql.checkData(0, 6, 5.000000000) + # TSIM: return -1 + # TSIM: endi + # TSIM: if $data07 != 6 then + tdLog.info('tdSql.checkData(0, 7, "6")') + tdSql.checkData(0, 7, "6") + tdLog.info('tdSql.checkData(0, 8, 7)') + tdSql.checkData(0, 8, 7) + tdLog.info('tdSql.checkData(0, 9, 8)') + tdSql.checkData(0, 9, 8) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql alter table $mt drop tag tgcol3 + tdLog.info('alter table %s drop tag tgcol3' % (mt)) + tdSql.execute('alter table %s drop tag tgcol3' % (mt)) + # TSIM: sql alter table $mt drop tag tgcol4 + tdLog.info('alter table %s drop tag tgcol4' % (mt)) + tdSql.execute('alter table %s drop tag tgcol4' % (mt)) + # TSIM: sql alter table $mt drop tag tgcol6 + tdLog.info('alter table %s drop tag tgcol6' % (mt)) + tdSql.execute('alter table %s drop tag tgcol6' % (mt)) + tdLog.info('alter table %s drop tag tgcol8' % (mt)) + tdSql.execute('alter table %s drop tag tgcol8' % (mt)) # TSIM: # TSIM: sleep 5000 # TSIM: @@ -1556,7 +1641,7 @@ class TDTestCase: # TSIM: print =============== step14 tdLog.info('=============== step14') # TSIM: $i = 14 - i = 14 + i = 20 # TSIM: $mt = $mtPrefix . $i mt = "%s%d" % (mtPrefix, i) # TSIM: $tb = $tbPrefix . $i diff --git a/tests/pytest/tag_lite/unsignedBigint.py b/tests/pytest/tag_lite/unsignedBigint.py new file mode 100644 index 0000000000000000000000000000000000000000..032dacd366437c378b0b8c9b553d6495966cd563 --- /dev/null +++ b/tests/pytest/tag_lite/unsignedBigint.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: + # TSIM: system sh/deploy.sh -n dnode1 -i 1 + # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + # TSIM: + # TSIM: $dbPrefix = ta_sm_db + # TSIM: $tbPrefix = ta_sm_tb + tbPrefix = "ta_sm_tb" + # TSIM: $mtPrefix = ta_sm_mt + mtPrefix = "ta_sm_mt" + # TSIM: $tbNum = 10 + tbNum = 10 + # TSIM: $rowNum = 20 + rowNum = 20 + # TSIM: $totalNum = 200 + totalNum = 200 + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: $i = 0 + i = 0 + # TSIM: $db = $dbPrefix . $i + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol + # bigint unsigned) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol bigint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol bigint unsigned)' % + (mt)) + # TSIM: + # TSIM: $i = 0 + i = 0 + # TSIM: while $i < 5 + while (i < 5): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 0 ) + tdLog.info('create table %s using %s tags( 0 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 0 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: while $i < 10 + while (i < 10): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 1 ) + tdLog.info('create table %s using %s tags( 1 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sleep 100 + # TSIM: sql select * from $tb + tdLog.info('select * from %s' % (tb)) + tdSql.query('select * from %s' % (tb)) + # TSIM: if $rows != $rowNum then + tdLog.info('tdSql.checkRow($rowNum)') + tdSql.checkRows(rowNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (tb)) + tdSql.query('select * from %s where ts < now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts <= now + 4m + tdLog.info('select * from %s where ts <= now + 4m' % (tb)) + tdSql.query('select * from %s where ts <= now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (tb)) + tdSql.query('select * from %s where ts > now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts >= now + 4m + tdLog.info('select * from %s where ts >= now + 4m' % (tb)) + tdSql.query('select * from %s where ts >= now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m and ts > now + 5m + tdLog.info( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > 100000 and ts < 100000 + tdLog.info('select * from %s where ts > 100000 and ts < 100000' % (tb)) + tdSql.query( + 'select * from %s where ts > 100000 and ts < 100000' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 3m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts > now + 5m and + # ts < now + 6m + tdLog.info( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: sql select * from $mt + tdLog.info('select * from %s' % (mt)) + tdSql.query('select * from %s' % (mt)) + # TSIM: if $rows != $totalNum then + tdLog.info('tdSql.checkRow($totalNum)') + tdSql.checkRows(totalNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (mt)) + tdSql.query('select * from %s where ts < now + 4m' % (mt)) + # TSIM: if $rows != 50 then + tdLog.info('tdSql.checkRow(50)') + tdSql.checkRows(50) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (mt)) + tdSql.query('select * from %s where ts > now + 4m' % (mt)) + # TSIM: if $rows != 150 then + tdLog.info('tdSql.checkRow(150)') + tdSql.checkRows(150) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts = now + 4m + tdLog.info('select * from %s where ts = now + 4m' % (mt)) + tdSql.query('select * from %s where ts = now + 4m' % (mt)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 10 then + tdLog.info('tdSql.checkRow(10)') + tdSql.checkRows(10) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql select * from $mt where ts > now + 4m and tgcol = 1 + tdLog.info('select * from %s where ts > now + 4m and tgcol = 1' % (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol = 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 1 + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol = 0 + tdLog.info('select * from %s where ts < now + 4m and tgcol = 0' % (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol = 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m and + # tgcol <> 0 + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 0 and ts + # < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 50 then + tdLog.info('tdSql.checkData(0, 0, 50)') + tdSql.checkData(0, 0, 50) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step9 + tdLog.info('=============== step9') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step10 + tdLog.info('=============== step10') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 group + # by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step11 + tdLog.info('=============== step11') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + # group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 25 then + tdLog.info('tdSql.checkData(0, 0, 25)') + tdSql.checkData(0, 0, 25) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: + # TSIM: print =============== step12 + tdLog.info('=============== step12') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) group by + # tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data01 != 100 then + tdLog.info('tdSql.checkData(0, 1, 100)') + tdSql.checkData(0, 1, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== clear + tdLog.info('=============== clear') + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev bigint unsigned)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 64) - 1)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1)) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 64) - 2)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (0)) + tdSql.execute( + 'CREATE TABLE if not exists dev_003 using st tags(%s)' % ('NULL')) + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/unsignedInt.py b/tests/pytest/tag_lite/unsignedInt.py new file mode 100644 index 0000000000000000000000000000000000000000..6efe12edf110f70d9c2c0187429a0696372bae49 --- /dev/null +++ b/tests/pytest/tag_lite/unsignedInt.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: + # TSIM: system sh/deploy.sh -n dnode1 -i 1 + # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + # TSIM: + # TSIM: $dbPrefix = ta_sm_db + # TSIM: $tbPrefix = ta_sm_tb + tbPrefix = "ta_sm_tb" + # TSIM: $mtPrefix = ta_sm_mt + mtPrefix = "ta_sm_mt" + # TSIM: $tbNum = 10 + tbNum = 10 + # TSIM: $rowNum = 20 + rowNum = 20 + # TSIM: $totalNum = 200 + totalNum = 200 + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: $i = 0 + i = 0 + # TSIM: $db = $dbPrefix . $i + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol + # int unsigned) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol int unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol int unsigned)' % + (mt)) + # TSIM: + # TSIM: $i = 0 + i = 0 + # TSIM: while $i < 5 + while (i < 5): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 0 ) + tdLog.info('create table %s using %s tags( 0 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 0 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: while $i < 10 + while (i < 10): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 1 ) + tdLog.info('create table %s using %s tags( 1 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sleep 100 + # TSIM: sql select * from $tb + tdLog.info('select * from %s' % (tb)) + tdSql.query('select * from %s' % (tb)) + # TSIM: if $rows != $rowNum then + tdLog.info('tdSql.checkRow($rowNum)') + tdSql.checkRows(rowNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (tb)) + tdSql.query('select * from %s where ts < now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts <= now + 4m + tdLog.info('select * from %s where ts <= now + 4m' % (tb)) + tdSql.query('select * from %s where ts <= now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (tb)) + tdSql.query('select * from %s where ts > now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts >= now + 4m + tdLog.info('select * from %s where ts >= now + 4m' % (tb)) + tdSql.query('select * from %s where ts >= now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m and ts > now + 5m + tdLog.info( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > 100000 and ts < 100000 + tdLog.info('select * from %s where ts > 100000 and ts < 100000' % (tb)) + tdSql.query( + 'select * from %s where ts > 100000 and ts < 100000' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 3m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts > now + 5m and + # ts < now + 6m + tdLog.info( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: sql select * from $mt + tdLog.info('select * from %s' % (mt)) + tdSql.query('select * from %s' % (mt)) + # TSIM: if $rows != $totalNum then + tdLog.info('tdSql.checkRow($totalNum)') + tdSql.checkRows(totalNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (mt)) + tdSql.query('select * from %s where ts < now + 4m' % (mt)) + # TSIM: if $rows != 50 then + tdLog.info('tdSql.checkRow(50)') + tdSql.checkRows(50) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (mt)) + tdSql.query('select * from %s where ts > now + 4m' % (mt)) + # TSIM: if $rows != 150 then + tdLog.info('tdSql.checkRow(150)') + tdSql.checkRows(150) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts = now + 4m + tdLog.info('select * from %s where ts = now + 4m' % (mt)) + tdSql.query('select * from %s where ts = now + 4m' % (mt)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 10 then + tdLog.info('tdSql.checkRow(10)') + tdSql.checkRows(10) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql select * from $mt where ts > now + 4m and tgcol = 1 + tdLog.info('select * from %s where ts > now + 4m and tgcol = 1' % (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol = 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 1 + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol = 0 + tdLog.info('select * from %s where ts < now + 4m and tgcol = 0' % (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol = 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m and + # tgcol <> 0 + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 0 and ts + # < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 50 then + tdLog.info('tdSql.checkData(0, 0, 50)') + tdSql.checkData(0, 0, 50) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step9 + tdLog.info('=============== step9') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step10 + tdLog.info('=============== step10') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 group + # by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step11 + tdLog.info('=============== step11') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + # group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 25 then + tdLog.info('tdSql.checkData(0, 0, 25)') + tdSql.checkData(0, 0, 25) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: + # TSIM: print =============== step12 + tdLog.info('=============== step12') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) group by + # tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data01 != 100 then + tdLog.info('tdSql.checkData(0, 1, 100)') + tdSql.checkData(0, 1, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== clear + tdLog.info('=============== clear') + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev int unsigned)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 32) - 1)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1)) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 32) - 2)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (0)) + tdSql.execute( + 'CREATE TABLE if not exists dev_003 using st tags(%s)' % ('NULL')) + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/unsignedSmallint.py b/tests/pytest/tag_lite/unsignedSmallint.py new file mode 100644 index 0000000000000000000000000000000000000000..59cb33dc18eb74b27e9a916e921e10cbabf1b4eb --- /dev/null +++ b/tests/pytest/tag_lite/unsignedSmallint.py @@ -0,0 +1,603 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: + # TSIM: system sh/deploy.sh -n dnode1 -i 1 + # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + # TSIM: + # TSIM: $dbPrefix = ta_sm_db + # TSIM: $tbPrefix = ta_sm_tb + tbPrefix = "ta_sm_tb" + # TSIM: $mtPrefix = ta_sm_mt + mtPrefix = "ta_sm_mt" + # TSIM: $tbNum = 10 + tbNum = 10 + # TSIM: $rowNum = 20 + rowNum = 20 + # TSIM: $totalNum = 200 + totalNum = 200 + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: $i = 0 + i = 0 + # TSIM: $db = $dbPrefix . $i + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol + # smallint unsigned) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol smallint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol smallint unsigned)' % + (mt)) + # TSIM: + # TSIM: $i = 0 + i = 0 + # TSIM: while $i < 5 + while (i < 5): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 0 ) + tdLog.info('create table %s using %s tags( 0 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 0 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: while $i < 10 + while (i < 10): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 1 ) + tdLog.info('create table %s using %s tags( 1 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sleep 100 + # TSIM: sql select * from $tb + tdLog.info('select * from %s' % (tb)) + tdSql.query('select * from %s' % (tb)) + # TSIM: if $rows != $rowNum then + tdLog.info('tdSql.checkRow($rowNum)') + tdSql.checkRows(rowNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (tb)) + tdSql.query('select * from %s where ts < now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts <= now + 4m + tdLog.info('select * from %s where ts <= now + 4m' % (tb)) + tdSql.query('select * from %s where ts <= now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (tb)) + tdSql.query('select * from %s where ts > now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts >= now + 4m + tdLog.info('select * from %s where ts >= now + 4m' % (tb)) + tdSql.query('select * from %s where ts >= now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m and ts > now + 5m + tdLog.info( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > 100000 and ts < 100000 + tdLog.info('select * from %s where ts > 100000 and ts < 100000' % (tb)) + tdSql.query( + 'select * from %s where ts > 100000 and ts < 100000' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 3m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts > now + 5m and + # ts < now + 6m + tdLog.info( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: sql select * from $mt + tdLog.info('select * from %s' % (mt)) + tdSql.query('select * from %s' % (mt)) + # TSIM: if $rows != $totalNum then + tdLog.info('tdSql.checkRow($totalNum)') + tdSql.checkRows(totalNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (mt)) + tdSql.query('select * from %s where ts < now + 4m' % (mt)) + # TSIM: if $rows != 50 then + tdLog.info('tdSql.checkRow(50)') + tdSql.checkRows(50) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (mt)) + tdSql.query('select * from %s where ts > now + 4m' % (mt)) + # TSIM: if $rows != 150 then + tdLog.info('tdSql.checkRow(150)') + tdSql.checkRows(150) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts = now + 4m + tdLog.info('select * from %s where ts = now + 4m' % (mt)) + tdSql.query('select * from %s where ts = now + 4m' % (mt)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 10 then + tdLog.info('tdSql.checkRow(10)') + tdSql.checkRows(10) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql select * from $mt where ts > now + 4m and tgcol = 1 + tdLog.info('select * from %s where ts > now + 4m and tgcol = 1' % (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol = 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 1 + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol = 0 + tdLog.info('select * from %s where ts < now + 4m and tgcol = 0' % (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol = 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m and + # tgcol <> 0 + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 0 and ts + # < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 50 then + tdLog.info('tdSql.checkData(0, 0, 50)') + tdSql.checkData(0, 0, 50) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step9 + tdLog.info('=============== step9') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step10 + tdLog.info('=============== step10') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 group + # by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step11 + tdLog.info('=============== step11') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + # group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 25 then + tdLog.info('tdSql.checkData(0, 0, 25)') + tdSql.checkData(0, 0, 25) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: + # TSIM: print =============== step12 + tdLog.info('=============== step12') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) group by + # tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data01 != 100 then + tdLog.info('tdSql.checkData(0, 1, 100)') + tdSql.checkData(0, 1, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== clear + tdLog.info('=============== clear') + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev smallint unsigned)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 16)-1)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1)) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 16) - 2)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (0)) + tdSql.execute( + 'CREATE TABLE if not exists dev_003 using st tags(%s)' % ('NULL')) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/unsignedTinyint.py b/tests/pytest/tag_lite/unsignedTinyint.py new file mode 100644 index 0000000000000000000000000000000000000000..1250b08d1b8dd4d3cb5f61af424576146ab70967 --- /dev/null +++ b/tests/pytest/tag_lite/unsignedTinyint.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + # TSIM: system sh/stop_dnodes.sh + # TSIM: + # TSIM: + # TSIM: system sh/deploy.sh -n dnode1 -i 1 + # TSIM: system sh/cfg.sh -n dnode1 -c walLevel -v 0 + # TSIM: system sh/exec.sh -n dnode1 -s start + # TSIM: + # TSIM: sleep 3000 + # TSIM: sql connect + # TSIM: + # TSIM: print ======================== dnode1 start + tdLog.info('======================== dnode1 start') + # TSIM: + # TSIM: $dbPrefix = ta_sm_db + # TSIM: $tbPrefix = ta_sm_tb + tbPrefix = "ta_sm_tb" + # TSIM: $mtPrefix = ta_sm_mt + mtPrefix = "ta_sm_mt" + # TSIM: $tbNum = 10 + tbNum = 10 + # TSIM: $rowNum = 20 + rowNum = 20 + # TSIM: $totalNum = 200 + totalNum = 200 + # TSIM: + # TSIM: print =============== step1 + tdLog.info('=============== step1') + # TSIM: $i = 0 + i = 0 + # TSIM: $db = $dbPrefix . $i + # TSIM: $mt = $mtPrefix . $i + mt = "%s%d" % (mtPrefix, i) + # TSIM: + # TSIM: sql create database $db + # TSIM: sql use $db + # TSIM: sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol + # tinyint unsigned) + tdLog.info( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol tinyint unsigned)' % + (mt)) + tdSql.execute( + 'create table %s (ts timestamp, tbcol int) TAGS(tgcol tinyint unsigned)' % + (mt)) + # TSIM: + # TSIM: $i = 0 + i = 0 + # TSIM: while $i < 5 + while (i < 5): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 0 ) + tdLog.info('create table %s using %s tags( 0 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 0 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: while $i < 10 + while (i < 10): + # TSIM: $tb = $tbPrefix . $i + tb = "%s%d" % (tbPrefix, i) + # TSIM: sql create table $tb using $mt tags( 1 ) + tdLog.info('create table %s using %s tags( 1 )' % (tb, mt)) + tdSql.execute('create table %s using %s tags( 1 )' % (tb, mt)) + # TSIM: $x = 0 + x = 0 + # TSIM: while $x < $rowNum + while (x < rowNum): + # TSIM: $ms = $x . m + ms = "%dm" % x + # TSIM: sql insert into $tb values (now + $ms , $x ) + tdLog.info( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + tdSql.execute( + 'insert into %s values (now + %s , %d )' % + (tb, ms, x)) + # TSIM: $x = $x + 1 + x = x + 1 + # TSIM: endw + # TSIM: $i = $i + 1 + i = i + 1 + # TSIM: endw + # TSIM: + # TSIM: print =============== step2 + tdLog.info('=============== step2') + # TSIM: sleep 100 + # TSIM: sql select * from $tb + tdLog.info('select * from %s' % (tb)) + tdSql.query('select * from %s' % (tb)) + # TSIM: if $rows != $rowNum then + tdLog.info('tdSql.checkRow($rowNum)') + tdSql.checkRows(rowNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (tb)) + tdSql.query('select * from %s where ts < now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts <= now + 4m + tdLog.info('select * from %s where ts <= now + 4m' % (tb)) + tdSql.query('select * from %s where ts <= now + 4m' % (tb)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (tb)) + tdSql.query('select * from %s where ts > now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts >= now + 4m + tdLog.info('select * from %s where ts >= now + 4m' % (tb)) + tdSql.query('select * from %s where ts >= now + 4m' % (tb)) + # TSIM: if $rows != 15 then + tdLog.info('tdSql.checkRow(15)') + tdSql.checkRows(15) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts < now + 4m and ts > now + 5m + tdLog.info( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + tdSql.query( + 'select * from %s where ts < now + 4m and ts > now + 5m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > 100000 and ts < 100000 + tdLog.info('select * from %s where ts > 100000 and ts < 100000' % (tb)) + tdSql.query( + 'select * from %s where ts > 100000 and ts < 100000' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts < now + 3m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 3m' % + (tb)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $tb where ts > now + 4m and ts > now + 5m and + # ts < now + 6m + tdLog.info( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts > now + 5m and ts < now + 6m' % + (tb)) + # TSIM: if $rows != 1 then + tdLog.info('tdSql.checkRow(1)') + tdSql.checkRows(1) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step3 + tdLog.info('=============== step3') + # TSIM: sql select * from $mt + tdLog.info('select * from %s' % (mt)) + tdSql.query('select * from %s' % (mt)) + # TSIM: if $rows != $totalNum then + tdLog.info('tdSql.checkRow($totalNum)') + tdSql.checkRows(totalNum) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: sql select * from $mt where ts < now + 4m + tdLog.info('select * from %s where ts < now + 4m' % (mt)) + tdSql.query('select * from %s where ts < now + 4m' % (mt)) + # TSIM: if $rows != 50 then + tdLog.info('tdSql.checkRow(50)') + tdSql.checkRows(50) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m + tdLog.info('select * from %s where ts > now + 4m' % (mt)) + tdSql.query('select * from %s where ts > now + 4m' % (mt)) + # TSIM: if $rows != 150 then + tdLog.info('tdSql.checkRow(150)') + tdSql.checkRows(150) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts = now + 4m + tdLog.info('select * from %s where ts = now + 4m' % (mt)) + tdSql.query('select * from %s where ts = now + 4m' % (mt)) + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 10 then + tdLog.info('tdSql.checkRow(10)') + tdSql.checkRows(10) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step4 + tdLog.info('=============== step4') + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 1 + tdLog.info('select * from %s where tgcol = 1' % (mt)) + tdSql.query('select * from %s where tgcol = 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 1 + tdLog.info('select * from %s where tgcol <> 1' % (mt)) + tdSql.query('select * from %s where tgcol <> 1' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol = 0 + tdLog.info('select * from %s where tgcol = 0' % (mt)) + tdSql.query('select * from %s where tgcol = 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where tgcol <> 0 + tdLog.info('select * from %s where tgcol <> 0' % (mt)) + tdSql.query('select * from %s where tgcol <> 0' % (mt)) + # TSIM: if $rows != 100 then + tdLog.info('tdSql.checkRow(100)') + tdSql.checkRows(100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step5 + tdLog.info('=============== step5') + # TSIM: sql select * from $mt where ts > now + 4m and tgcol = 1 + tdLog.info('select * from %s where ts > now + 4m and tgcol = 1' % (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol = 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 1 + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 1' % + (mt)) + # TSIM: if $rows != 75 then + tdLog.info('tdSql.checkRow(75)') + tdSql.checkRows(75) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol = 0 + tdLog.info('select * from %s where ts < now + 4m and tgcol = 0' % (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts < now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts < now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol = 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol = 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts <= now + 4m and tgcol <> 0 + tdLog.info( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts <= now + 4m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 25 then + tdLog.info('tdSql.checkRow(25)') + tdSql.checkRows(25) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and ts < now + 5m and + # tgcol <> 0 + tdLog.info( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and ts < now + 5m and tgcol <> 0' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: sql select * from $mt where ts > now + 4m and tgcol <> 0 and ts + # < now + 5m + tdLog.info( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + tdSql.query( + 'select * from %s where ts > now + 4m and tgcol <> 0 and ts < now + 5m' % + (mt)) + # TSIM: if $rows != 5 then + tdLog.info('tdSql.checkRow(5)') + tdSql.checkRows(5) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step6 + tdLog.info('=============== step6') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 200 then + tdLog.info('tdSql.checkData(0, 0, 200)') + tdSql.checkData(0, 0, 200) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step7 + tdLog.info('=============== step7') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step8 + tdLog.info('=============== step8') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 50 then + tdLog.info('tdSql.checkData(0, 0, 50)') + tdSql.checkData(0, 0, 50) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step9 + tdLog.info('=============== step9') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step10 + tdLog.info('=============== step10') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where tgcol = 1 group + # by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where tgcol = 1 group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 100 then + tdLog.info('tdSql.checkData(0, 0, 100)') + tdSql.checkData(0, 0, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== step11 + tdLog.info('=============== step11') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m + # group by tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s where ts < now + 4m group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data00 != 25 then + tdLog.info('tdSql.checkData(0, 0, 25)') + tdSql.checkData(0, 0, 25) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: + # TSIM: print =============== step12 + tdLog.info('=============== step12') + # TSIM: sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), + # max(tbcol), first(tbcol), last(tbcol) from $mt interval(1d) group by + # tgcol + tdLog.info( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + tdSql.query( + 'select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from %s interval(1d) group by tgcol' % + (mt)) + # TSIM: print $data00 $data01 $data02 $data03 $data04 $data05 $data06 + tdLog.info('$data00 $data01 $data02 $data03 $data04 $data05 $data06') + # TSIM: if $data01 != 100 then + tdLog.info('tdSql.checkData(0, 1, 100)') + tdSql.checkData(0, 1, 100) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: print =============== clear + tdLog.info('=============== clear') + # TSIM: sql drop database $db + tdLog.info('drop database db') + tdSql.execute('drop database db') + # TSIM: sql show databases + tdLog.info('show databases') + tdSql.query('show databases') + # TSIM: if $rows != 0 then + tdLog.info('tdSql.checkRow(0)') + tdSql.checkRows(0) + # TSIM: return -1 + # TSIM: endi + # TSIM: + # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT +# convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev tinyint unsigned)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 8) - 1)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1)) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 8) - 2)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (0)) + tdSql.execute( + 'CREATE TABLE if not exists dev_003 using st tags(%s)' % ('NULL')) + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/insert-interlace.json b/tests/pytest/tools/insert-interlace.json new file mode 100644 index 0000000000000000000000000000000000000000..d4767ad0640c8b2a1528fc24e681c359b719a4b9 --- /dev/null +++ b/tests/pytest/tools/insert-interlace.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 5000, + "interlace_rows": 50, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 9, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 250, + "multi_thread_write_one_tbl": "no", + "interlace_rows": 80, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}], + "tags": [{"type": "INT", "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/insert-tblimit-tboffset.json b/tests/pytest/tools/insert-tblimit-tboffset.json new file mode 100644 index 0000000000000000000000000000000000000000..f3d3e864baaa5429248f57a6dd1ea6b5bc314bfb --- /dev/null +++ b/tests/pytest/tools/insert-tblimit-tboffset.json @@ -0,0 +1,59 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 33, + "childtable_offset": 33, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/insert-tblimit-tboffset0.json b/tests/pytest/tools/insert-tblimit-tboffset0.json new file mode 100644 index 0000000000000000000000000000000000000000..7dcb2e052745e545d1693d97ce28350d00745e55 --- /dev/null +++ b/tests/pytest/tools/insert-tblimit-tboffset0.json @@ -0,0 +1,59 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 20, + "childtable_offset": 0, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/insert-tblimit1-tboffset.json b/tests/pytest/tools/insert-tblimit1-tboffset.json new file mode 100644 index 0000000000000000000000000000000000000000..a33dc22d5d5e2c88c9b6220fbe53390dfcd4f1ce --- /dev/null +++ b/tests/pytest/tools/insert-tblimit1-tboffset.json @@ -0,0 +1,59 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 1, + "childtable_offset": 50, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}], + "tags": [{"type": "TINYINT", "count":1}] + }] + }] +} diff --git a/tests/pytest/tools/insert.json b/tests/pytest/tools/insert.json index c3fa78076b2a25f73ebc50f6a35bcc5afddb246d..996b91ed06f283fdcd968df9cafc4f58583cbb8d 100644 --- a/tests/pytest/tools/insert.json +++ b/tests/pytest/tools/insert.json @@ -1,50 +1,45 @@ { "filetype":"insert", "cfgdir": "/etc/taos", - "host": "127.0.0.1", - "port": 6030, - "user": "root", - "password": "taosdata", - "thread_count": 1, - "databases": [{ - "dbinfo": { - "name": "db01", - "replica": 1, - "days": 10, - "cache": 16, - "blocks": 8, - "precision": "ms", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 2, + "num_of_records_per_req": 10, + "thread_count_create_tbl": 4, + "databases": [{ + "dbinfo": { + "name": "db01", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", "update": 0, - "maxtablesPerVnode": 1000 - }, - "super_tables": [{ - "name": "stb01", - "childtable_count": 100, - "childtable_prefix": "stb01_", - "auto_create_table": "no", - "data_source": "rand", - "insert_mode": "taosc", - "insert_rate": 0, - "insert_rows": 1000, - "timestamp_step": 1000, - "start_timestamp": "2020-10-01 00:00:00.000", - "sample_format": "csv", - "sample_file": "/home/data/sample.csv", - "tags_file": "", - "columns": [{ - "type": "SMALLINT" - }, { - "type": "BOOL" - }, { - "type": "BINARY", - "len": 6 - }], - "tags": [{ - "type": "INT" - },{ - "type": "BINARY", - "len": 4 - }] - }] - }] + "maxtablesPerVnode": 1000 + }, + "super_tables": [{ + "name": "stb01", + "childtable_count": 3, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rate": 0, + "insert_rows": 20, + "timestamp_step": 1000, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "/home/data/sample.csv", + "tags_file": "", + "columns": [{ + "type": "INT" + }], + "tags": [{ + "type": "INT" + }] + }] + }] } diff --git a/tests/pytest/tools/query.json b/tests/pytest/tools/query.json new file mode 100644 index 0000000000000000000000000000000000000000..d486423865a7a21bdd4817b9b514131942806777 --- /dev/null +++ b/tests/pytest/tools/query.json @@ -0,0 +1,22 @@ +{ + "filetype": "query", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "no", + "databases": "test", + "query_times": 1, + "super_table_query": { + "stblname": "meters", + "query_interval": 10, + "threads": 8, + "sqls": [ + { + "sql": "select last_row(ts) from xxxx", + "result": "" + } + ] + } +} diff --git a/tests/pytest/tools/sampledata.csv b/tests/pytest/tools/sampledata.csv new file mode 100644 index 0000000000000000000000000000000000000000..01e79c32a8c99c557f0757da7cb6d65b3414466d --- /dev/null +++ b/tests/pytest/tools/sampledata.csv @@ -0,0 +1,3 @@ +1 +2 +3 diff --git a/tests/pytest/tools/taosdemo-sampledata.json b/tests/pytest/tools/taosdemo-sampledata.json new file mode 100644 index 0000000000000000000000000000000000000000..473c977773a097fd040b3821c3df806da8ef9c02 --- /dev/null +++ b/tests/pytest/tools/taosdemo-sampledata.json @@ -0,0 +1,39 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "confirm_parameter_prompt": "no", + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes" + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 20, + "childtable_limit": 10, + "childtable_offset": 0, + "childtable_prefix": "t_", + "auto_create_table": "no", + "data_source": "sample", + "insert_mode": "taosc", + "insert_rate": 0, + "insert_rows": 20, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "max_sql_len": 1048000, + "timestamp_step": 1000, + "start_timestamp": "2020-1-1 00:00:00", + "sample_format": "csv", + "sample_file": "./tools/sampledata.csv", + "columns": [{"type": "INT"}], + "tags": [{"type": "INT", "count":1}] + }] + }] + +} diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 28f451b6a08a43508352c75b0c498251c9afca54..1156cc2484799429f7f5fc689faee03453dad2ed 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -11,26 +11,16 @@ # -*- coding: utf-8 -*- -import sys import taos -import time -import datetime -import csv -import random import pandas as pd import argparse import os.path +import json class taosdemoPerformace: - def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay): + def __init__(self, commitID, dbName): self.commitID = commitID - self.dbName = dbName - self.createTableTime = createTableTime - self.insertRecordsTime = insertRecordsTime - self.recordsPerSecond = recordsPerSecond - self.avgDelay = avgDelay - self.maxDelay = maxDelay - self.minDelay = minDelay + self.dbName = dbName self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -39,8 +29,97 @@ class taosdemoPerformace: self.host, self.user, self.password, - self.config) + self.config) + self.insertDB = "insertDB"; + def generateJson(self): + db = { + "name": "%s" % self.insertDB, + "drop": "yes", + "replica": 1 + } + + stb = { + "name": "meters", + "child_table_exists":"no", + "childtable_count": 10000, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "batch_create_tbl_num": 10, + "insert_mode": "taosc", + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 100, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 4} + ], + "tags": [ + {"type": "INT", "count":1}, + {"type": "BINARY", "len": 16} + ] + } + + stables = [] + stables.append(stb) + + db = { + "dbinfo": db, + "super_tables": stables + } + + insert_data = { + "filetype": "insert", + "cfgdir": "/etc/taosperf", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 30000, + "databases": [db] + } + + insert_json_file = f"/tmp/insert.json" + + with open(insert_json_file, 'w') as f: + json.dump(insert_data, f) + return insert_json_file + + def getCMDOutput(self, cmd): + cmd = os.popen(cmd) + output = cmd.read() + cmd.close() + return output + + def insertData(self): + os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson()) + self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") + self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") + self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") + self.commitID = self.getCMDOutput("git rev-parse --short HEAD") + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'") + self.avgDelay = delay[:-4] + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'") + self.maxDelay = delay[:-4] + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'") + self.minDelay = delay[:-3] + + os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt") + def createTablesAndStoreData(self): cursor = self.conn.cursor() @@ -48,14 +127,15 @@ class taosdemoPerformace: cursor.execute("use %s" % self.dbName) cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") print("==================== taosdemo performance ====================") - print("create tables time: %f" % self.createTableTime) - print("insert records time: %f" % self.insertRecordsTime) - print("records per second: %f" % self.recordsPerSecond) - print("avg delay: %f" % self.avgDelay) - print("max delay: %f" % self.maxDelay) - print("min delay: %f" % self.minDelay) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay)) - cursor.execute("drop database if exists taosdemo_insert_test") + print("create tables time: %f" % float(self.createTableTime)) + print("insert records time: %f" % float(self.insertRecordsTime)) + print("records per second: %f" % float(self.recordsPerSecond)) + print("avg delay: %f" % float(self.avgDelay)) + print("max delay: %f" % float(self.maxDelay)) + print("min delay: %f" % float(self.minDelay)) + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % + (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay))) + cursor.execute("drop database if exists %s" % self.insertDB) cursor.close() @@ -64,7 +144,7 @@ if __name__ == '__main__': parser.add_argument( '-c', '--commit-id', - action='store', + action='store', type=str, help='git commit id (default: null)') parser.add_argument( @@ -74,46 +154,9 @@ if __name__ == '__main__': default='perf', type=str, help='Database name to be created (default: perf)') - parser.add_argument( - '-t', - '--create-table', - action='store', - type=float, - help='create table time') - parser.add_argument( - '-i', - '--insert-records', - action='store', - type=float, - help='insert records time') - parser.add_argument( - '-r', - '---records-per-second', - action='store', - type=float, - help='records per request') - parser.add_argument( - '-avg', - '---avg-delay', - action='store', - type=float, - help='avg delay') - parser.add_argument( - '-max', - '---max-delay', - action='store', - type=float, - help='max delay') - parser.add_argument( - '-min', - '---min-delay', - action='store', - type=float, - help='min delay') - args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second, - args.avg_delay, args.max_delay, args.min_delay) + perftest = taosdemoPerformace(args.commit_id, args.database_name) + perftest.insertData() perftest.createTablesAndStoreData() \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoTest.py b/tests/pytest/tools/taosdemoTest.py index 2a4a552c8fb87f9c5a3169afcad71793462b924f..ff5921be604f9fe911f1aa8b84efe230baf20e07 100644 --- a/tests/pytest/tools/taosdemoTest.py +++ b/tests/pytest/tools/taosdemoTest.py @@ -23,9 +23,10 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - - self.numberOfTables = 10000 + + self.numberOfTables = 1000 self.numberOfRecords = 100 + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -38,9 +39,10 @@ class TDTestCase: if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + buildPath = root[:len(root) - len("/build/bin")] break return buildPath + def run(self): tdSql.prepare() buildPath = self.getBuildPath() @@ -48,18 +50,21 @@ class TDTestCase: tdLog.exit("taosd not found!") else: tdLog.info("taosd found in %s" % buildPath) - binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemo -t %d -n %d -x" % (binPath,self.numberOfTables, self.numberOfRecords)) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -y -t %d -n %d" % + (binPath, self.numberOfTables, self.numberOfRecords)) tdSql.execute("use test") tdSql.query("select count(*) from meters") tdSql.checkData(0, 0, self.numberOfTables * self.numberOfRecords) - tdSql.query("select sum(f1) from test.meters interval(1h) sliding(30m)") + tdSql.query( + "select sum(col1) from test.meters interval(1h) sliding(30m)") tdSql.checkRows(2) - tdSql.query("select apercentile(f1, 1) from test.meters interval(10s)") - tdSql.checkRows(11) + tdSql.query( + "select apercentile(col1, 1) from test.meters interval(100s)") + tdSql.checkRows(1) tdSql.error("select loc, count(loc) from test.meters") diff --git a/tests/pytest/tools/taosdemoTest2.py b/tests/pytest/tools/taosdemoTest2.py deleted file mode 100644 index 7d5627be4363467e5e736760eea1e9f0799437d9..0000000000000000000000000000000000000000 --- a/tests/pytest/tools/taosdemoTest2.py +++ /dev/null @@ -1,64 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -from util.log import * -from util.cases import * -from util.sql import * -from util.dnodes import * -import threading -import time - - -class TDTestCase: - def init(self, conn, logSql): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) - - self.numberOfTables = 10 - self.numberOfRecords = 1000000 - - def insertDataAndAlterTable(self, threadID): - if(threadID == 0): - os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords)) - if(threadID == 1): - print("use test") - tdSql.execute("use test") - print("alter table test.meters add column f4 int") - tdSql.execute("alter table test.meters add column f4 int") - print("insert into test.t0 values (now, 1, 2, 3, 4)") - tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4)") - - def run(self): - tdSql.prepare() - - t1 = threading.Thread(target=self.insertDataAndAlterTable, args=(0, )) - t2 = threading.Thread(target=self.insertDataAndAlterTable, args=(1, )) - - t1.start() - time.sleep(2) - t2.start() - t1.join() - t2.join() - - tdSql.query("select count(*) from test.meters") - tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1) - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoTestInterlace.py b/tests/pytest/tools/taosdemoTestInterlace.py new file mode 100644 index 0000000000000000000000000000000000000000..4c551f327a2af5021c9460430e7ff30e393386bb --- /dev/null +++ b/tests/pytest/tools/taosdemoTestInterlace.py @@ -0,0 +1,75 @@ +################################################################## +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import subprocess + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + taosdemoCmd = "%staosdemo -f tools/insert-interlace.json -pp 2>&1 | grep sleep | wc -l" % binPath + sleepTimes = subprocess.check_output( + taosdemoCmd, shell=True).decode("utf-8") + print("sleep times: %d" % int(sleepTimes)) + + if (int(sleepTimes) != 16): + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected sleep times 16, actual %d" % + (caller.filename, caller.lineno, int(sleepTimes))) + + tdSql.execute("use db") + tdSql.query("select count(tbname) from db.stb") + tdSql.checkData(0, 0, 9) + tdSql.query("select count(*) from db.stb") + tdSql.checkData(0, 0, 2250) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoTestLimitOffset.py b/tests/pytest/tools/taosdemoTestLimitOffset.py new file mode 100644 index 0000000000000000000000000000000000000000..bce41e1c75817c2939d4c1104419771483a9a689 --- /dev/null +++ b/tests/pytest/tools/taosdemoTestLimitOffset.py @@ -0,0 +1,86 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numberOfTables = 10000 + self.numberOfRecords = 100 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + os.system("%staosdemo -f tools/insert-tblimit-tboffset.json" % binPath) + + tdSql.execute("use db") + tdSql.query("select count(tbname) from db.stb") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from db.stb") + tdSql.checkData(0, 0, 33000) + + os.system("%staosdemo -f tools/insert-tblimit-tboffset0.json" % binPath) + + tdSql.execute("reset query cache") + tdSql.execute("use db") + tdSql.query("select count(tbname) from db.stb") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from db.stb") + tdSql.checkData(0, 0, 20000) + + os.system("%staosdemo -f tools/insert-tblimit1-tboffset.json" % binPath) + + tdSql.execute("reset query cache") + tdSql.execute("use db") + tdSql.query("select count(tbname) from db.stb") + tdSql.checkData(0, 0, 100) + tdSql.query("select count(*) from db.stb") + tdSql.checkData(0, 0, 1000) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoTestQuery.py b/tests/pytest/tools/taosdemoTestQuery.py new file mode 100644 index 0000000000000000000000000000000000000000..bb2bb85052a9b21dc9181887622ec2019707256b --- /dev/null +++ b/tests/pytest/tools/taosdemoTestQuery.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import subprocess + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numberOfTables = 1000 + self.numberOfRecords = 100 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -y -t %d -n %d" % + (binPath, self.numberOfTables, self.numberOfRecords)) + print("Sleep 2 seconds..") + time.sleep(2) + os.system('%staosdemo -f tools/query.json ' % binPath) +# taosdemoCmd = '%staosdemo -f tools/query.json ' % binPath +# threads = subprocess.check_output( +# taosdemoCmd, shell=True).decode("utf-8") +# print("threads: %d" % int(threads)) + +# if (int(threads) != 8): +# caller = inspect.getframeinfo(inspect.stack()[0][0]) +# tdLog.exit( +# "%s(%d) failed: expected threads 8, actual %d" % +# (caller.filename, caller.lineno, int(threads))) +# + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoTestSampleData.py b/tests/pytest/tools/taosdemoTestSampleData.py new file mode 100644 index 0000000000000000000000000000000000000000..893c53984d87aa729111a840831c6f5343087085 --- /dev/null +++ b/tests/pytest/tools/taosdemoTestSampleData.py @@ -0,0 +1,68 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numberOfTables = 10000 + self.numberOfRecords = 100 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + tdSql.prepare() + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + os.system("%staosdemo -f tools/taosdemo-sampledata.json" % binPath) + + tdSql.execute("use db") + tdSql.query("select count(tbname) from db.stb") + tdSql.checkData(0, 0, 20) + tdSql.query("select count(*) from db.stb") + tdSql.checkData(0, 0, 200) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/taosdemoTestTblAlt.py b/tests/pytest/tools/taosdemoTestTblAlt.py new file mode 100644 index 0000000000000000000000000000000000000000..bb367817cf082a4a4b75f2deac6883d72d7ba145 --- /dev/null +++ b/tests/pytest/tools/taosdemoTestTblAlt.py @@ -0,0 +1,128 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import threading +import time + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numberOfTables = 10 + self.numberOfRecords = 1000000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def insertDataAndAlterTable(self, threadID): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + + if(threadID == 0): + os.system("%staosdemo -y -t %d -n %d" % + (binPath, self.numberOfTables, self.numberOfRecords)) + if(threadID == 1): + time.sleep(2) + print("use test") + while True: + try: + tdSql.execute("use test") + break + except Exception as e: + tdLog.info("use database test failed") + time.sleep(1) + continue + + # check if all the tables have heen created + while True: + try: + tdSql.query("show tables") + except Exception as e: + tdLog.info("show tables test failed") + time.sleep(1) + continue + + rows = tdSql.queryRows + print("number of tables: %d" % rows) + if(rows == self.numberOfTables): + break + time.sleep(1) + # check if there are any records in the last created table + while True: + print("query started") + try: + tdSql.query("select * from test.t9") + except Exception as e: + tdLog.info("select * test failed") + time.sleep(2) + continue + + rows = tdSql.queryRows + print("number of records: %d" % rows) + if(rows > 0): + break + time.sleep(1) + + print("alter table test.meters add column col10 int") + tdSql.execute("alter table test.meters add column col10 int") + print("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") + tdSql.execute("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") + + def run(self): + tdSql.prepare() + + t1 = threading.Thread(target=self.insertDataAndAlterTable, args=(0, )) + t2 = threading.Thread(target=self.insertDataAndAlterTable, args=(1, )) + + t1.start() + t2.start() + t1.join() + t2.join() + + time.sleep(3) + + tdSql.query("select count(*) from test.meters") + tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tools/lowaTest.py b/tests/pytest/tools/taosdemoTestWithJson.py similarity index 91% rename from tests/pytest/tools/lowaTest.py rename to tests/pytest/tools/taosdemoTestWithJson.py index 2b65dcf3eff1ed9ed7275fd774807cfa0318ec81..f57af9ce5c7ce890a0a93c6081702d93954af138 100644 --- a/tests/pytest/tools/lowaTest.py +++ b/tests/pytest/tools/taosdemoTestWithJson.py @@ -24,9 +24,6 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - self.numberOfTables = 10000 - self.numberOfRecords = 100 - def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -51,11 +48,11 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - os.system("yes | %staosdemox -f tools/insert.json" % binPath) + os.system("yes | %staosdemo -f tools/insert.json" % binPath) tdSql.execute("use db01") tdSql.query("select count(*) from stb01") - tdSql.checkData(0, 0, 100000) + tdSql.checkData(0, 0, 60) def stop(self): tdSql.close() diff --git a/tests/pytest/tools/taosdemoTestWithoutMetric.py b/tests/pytest/tools/taosdemoTestWithoutMetric.py new file mode 100644 index 0000000000000000000000000000000000000000..9687600563d8fed68c6f9c67643759a3dcfa9703 --- /dev/null +++ b/tests/pytest/tools/taosdemoTestWithoutMetric.py @@ -0,0 +1,72 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numberOfTables = 100 + self.numberOfRecords = 1000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + os.system("%staosdemo -N -y -t %d -n %d" % + (binPath, self.numberOfTables, self.numberOfRecords)) + + tdSql.query("show databases") + for i in range(18): + print(tdSql.getData(0, i) ) + tdSql.checkData(0, 2, self.numberOfTables) + + tdSql.execute("use test") + tdSql.query( + "select count(*) from test.t%d" % (self.numberOfTables -1)) + tdSql.checkData(0, 0, self.numberOfRecords) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 91c75efe9762a18042660423708e6dd9475166dd..6eaf4e18af53eac2b6e3c93749528993d8477e3d 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -136,7 +136,8 @@ class TDDnode: "monitorDebugFlag":"135", "udebugFlag":"135", "jnidebugFlag":"135", - "qdebugFlag":"135" + "qdebugFlag":"135", + "maxSQLLength":"1048576" } def init(self, path): diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 4f7607fc6c000d7ca55035e6973bfb5577a67964..16931cca333a3300b7d0d6831bbb51db0238b1d1 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -194,11 +194,16 @@ class TDSql: pstate = 0 pl = psutil.pids() for pid in pl: - if psutil.Process(pid).name == 'taosd': - pstate = 1 - break - if pstate: - tdLog.sleep(5) + try: + if psutil.Process(pid).name() == 'taosd': + print('have already started') + pstate = 1 + break + except psutil.NoSuchProcess: + pass + if pstate == state :break + if state or pstate: + tdLog.sleep(1) continue pstate = 0 break @@ -219,9 +224,9 @@ class TDSql: tdLog.info("dir: %s is empty, expect: empty" %dir) else: if state : - tdLog.info("dir: %s is empty, expect: not empty" %dir) + tdLog.info("dir: %s is not empty, expect: not empty" %dir) else: - tdLog.exit("dir: %s is empty, expect: empty" %dir) + tdLog.exit("dir: %s is not empty, expect: empty" %dir) else: tdLog.exit("dir: %s doesn't exist" %dir) def createDir(self, dir): diff --git a/tests/script/general/alter/cached_schema_after_alter.sim b/tests/script/general/alter/cached_schema_after_alter.sim index 2d049ec5955e281d866eaaa15abf06583909c87d..96ee4390845450d53508cc90c48a3148a0a827dd 100644 --- a/tests/script/general/alter/cached_schema_after_alter.sim +++ b/tests/script/general/alter/cached_schema_after_alter.sim @@ -3,7 +3,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $db = csaa_db @@ -53,10 +53,10 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -sleep 5000 +sleep 3000 sql connect sql use $db diff --git a/tests/script/general/alter/count.sim b/tests/script/general/alter/count.sim index 157b4c137198b70a9e7ad3b1f64e5b4b06a90415..fc936668b8ea08f9cd08874ad98668a4d8904315 100644 --- a/tests/script/general/alter/count.sim +++ b/tests/script/general/alter/count.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 print ========= start dnode1 as master system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -141,9 +141,9 @@ endi print ============= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb if $data00 != 24 then @@ -250,9 +250,9 @@ endi print ============== step18 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 #sql select count(g) from tb #if $data00 != 12 then diff --git a/tests/script/general/alter/dnode.sim b/tests/script/general/alter/dnode.sim index 73a095ec054320d0373f137250193bc4d8230234..7b31218fc231cfdbb79ca97573cfc6f6f149037d 100644 --- a/tests/script/general/alter/dnode.sim +++ b/tests/script/general/alter/dnode.sim @@ -4,14 +4,14 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 sql alter dnode 1 resetlog sql alter dnode 1 monitor 1 -sleep 5000 +sleep 3000 sql select * from log.dn if $rows <= 0 then return -1 diff --git a/tests/script/general/alter/import.sim b/tests/script/general/alter/import.sim index 76388a26f2b15d1ce9a86331ab2e70cb29a88480..aef0a258b24563e915cd8aa3dd42f6623a29170a 100644 --- a/tests/script/general/alter/import.sim +++ b/tests/script/general/alter/import.sim @@ -7,7 +7,7 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 print ========= start dnode1 as master system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/alter/insert1.sim b/tests/script/general/alter/insert1.sim index 3b16214465c15133aecc3e2c5977996cbaa22b5f..12ab09beb989dd963a9e8c9c3ff5926e78d8b0ac 100644 --- a/tests/script/general/alter/insert1.sim +++ b/tests/script/general/alter/insert1.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -940,9 +940,9 @@ endi print ======== step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql select * from tb order by ts asc if $rows != 8 then diff --git a/tests/script/general/alter/insert2.sim b/tests/script/general/alter/insert2.sim index 28948b69d231952f3baae0d79a75c1f2c9c6fb60..dcd9f500304f906ddddb33bd1a04c5943c232d49 100644 --- a/tests/script/general/alter/insert2.sim +++ b/tests/script/general/alter/insert2.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -608,9 +608,9 @@ sql_error alter table tb drop column a print ======== step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql select * from tb order by ts desc if $rows != 7 then diff --git a/tests/script/general/alter/metrics.sim b/tests/script/general/alter/metrics.sim index 5ed8050b96b90d679fcb85ada06c6065b8f0735a..fd0b210cd1b452b2a35ebcd9f74aec98c3817b03 100644 --- a/tests/script/general/alter/metrics.sim +++ b/tests/script/general/alter/metrics.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -367,9 +367,9 @@ endi print ======== step9 print ======== step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use d2 sql describe tb diff --git a/tests/script/general/alter/table.sim b/tests/script/general/alter/table.sim index 5a72fc925619e1a0ab40289056f6af355b995ca9..06704eeca6b3149b47ddc2ffb90aaab9df934bd8 100644 --- a/tests/script/general/alter/table.sim +++ b/tests/script/general/alter/table.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -318,9 +318,9 @@ endi print ======== step9 print ======== step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use d1 sql describe tb diff --git a/tests/script/general/cache/new_metrics.sim b/tests/script/general/cache/new_metrics.sim index 84f9e40de39b6946984204f4e3cb6f7e395af0e2..eb9b042483251856c2ac7b3f973ab31176a8ff56 100644 --- a/tests/script/general/cache/new_metrics.sim +++ b/tests/script/general/cache/new_metrics.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 @@ -94,8 +94,9 @@ while $i < 10 $i = $i + 1 endw -print ==> sleep 8 seconds to renew cache -sleep 8000 +print ==> sleep 1 seconds to renew cache +sql reset query cache +sleep 1000 print =============== step5 diff --git a/tests/script/general/cache/restart_metrics.sim b/tests/script/general/cache/restart_metrics.sim index f24768859fe8b771b9c03e0ea328d0066174909d..a1b2365b2aef739d7bdf97ba82b757648285fe97 100644 --- a/tests/script/general/cache/restart_metrics.sim +++ b/tests/script/general/cache/restart_metrics.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -49,13 +49,13 @@ endi print =============== step2 system sh/exec.sh -n dnode1 -s stop -sleep 5000 +sleep 3000 system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start print =============== step3 -print ==> sleep 8 seconds to renew cache +print ==> sleep 1 seconds to renew cache sql reset query cache sleep 1000 diff --git a/tests/script/general/cache/restart_table.sim b/tests/script/general/cache/restart_table.sim index 8697c26bc3d7708a4eca1c3480d6cdb248c072dc..1f7d982a2848a5df443f32a8c2b5c1edb97874d2 100644 --- a/tests/script/general/cache/restart_table.sim +++ b/tests/script/general/cache/restart_table.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -33,16 +33,15 @@ endi print =============== step2 system sh/exec.sh -n dnode1 -s stop -sleep 5000 +sleep 3000 system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start print =============== step3 -print ==> sleep 8 seconds to renew cache -sleep 3000 +print ==> sleep 1 seconds to renew cache sql reset query cache -sleep 18000 +sleep 1000 print =============== step4 sql create database $db diff --git a/tests/script/general/column/commit.sim b/tests/script/general/column/commit.sim index c574db1aa9dd28139251b4aa0433200e5fde230a..008bec3bf9dc6c0bd8ca0a4c2b5816a9a22c93b2 100644 --- a/tests/script/general/column/commit.sim +++ b/tests/script/general/column/commit.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -89,9 +89,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/column/metrics.sim b/tests/script/general/column/metrics.sim index 673b66c9e266a68171161c88d3e2dfadd9d446da..580e2320cd139af4bd9174f7eecef6dbdef97396 100644 --- a/tests/script/general/column/metrics.sim +++ b/tests/script/general/column/metrics.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -157,9 +157,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/column/table.sim b/tests/script/general/column/table.sim index aec0dc3c75031f9c4e167ffae5511ff34e4ec886..46d5de1e82f957ac8ccd223ae28f80286a27a827 100644 --- a/tests/script/general/column/table.sim +++ b/tests/script/general/column/table.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -129,9 +129,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print ============== step5 diff --git a/tests/script/general/compress/commitlog.sim b/tests/script/general/compress/commitlog.sim index b5d653fe83e94045c6f5f7ce7f58104504ad5330..e8eab6ed0cd9b5b6c2ea70042bd0f24e92c512a6 100644 --- a/tests/script/general/compress/commitlog.sim +++ b/tests/script/general/compress/commitlog.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c comp -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -87,9 +87,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/compress/compress.sim b/tests/script/general/compress/compress.sim index 6975f879968107bd142a3595869fc120bc1daa02..cecfe61229667086d83e963b8833ae9b0fabc2da 100644 --- a/tests/script/general/compress/compress.sim +++ b/tests/script/general/compress/compress.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c comp -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -82,9 +82,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/compress/compress2.sim b/tests/script/general/compress/compress2.sim index cf96f572ace4a693b1c5640986510d2f5a4d9d74..1e6868eaa6382218e4f4d1b8fb3f69c8762f77c8 100644 --- a/tests/script/general/compress/compress2.sim +++ b/tests/script/general/compress/compress2.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c comp -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -82,9 +82,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step5 diff --git a/tests/script/general/compress/uncompress.sim b/tests/script/general/compress/uncompress.sim index 13d288451c8031154f2c62fe4e5f8670fad8517a..49945dcb79ef69b31aa8cc4e8b0009e2ceb691f1 100644 --- a/tests/script/general/compress/uncompress.sim +++ b/tests/script/general/compress/uncompress.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c comp -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -81,9 +81,9 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 -system sh/exec.sh -n dnode1 -s start sleep 3000 +system sh/exec.sh -n dnode1 -s start +sleep 2000 print =============== step5 diff --git a/tests/script/general/compute/avg.sim b/tests/script/general/compute/avg.sim index 52a270bc6d9b1581fb210693f74cce9ecc73b3fa..db452b0344b6cbe14f313e613e737415780aeb27 100644 --- a/tests/script/general/compute/avg.sim +++ b/tests/script/general/compute/avg.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_av_db diff --git a/tests/script/general/compute/bottom.sim b/tests/script/general/compute/bottom.sim index 415bd36e2e0417e8f9da2989e21f1a1908e01cfd..7af67ecbf0237bd591d84a243fd88b80e167e4f7 100644 --- a/tests/script/general/compute/bottom.sim +++ b/tests/script/general/compute/bottom.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_bo_db diff --git a/tests/script/general/compute/count.sim b/tests/script/general/compute/count.sim index d8aeb18752a3fee18471a3402d259142486ecc1a..cf84918f5b7c2bd5a396b6457c75b4c59b62bb19 100644 --- a/tests/script/general/compute/count.sim +++ b/tests/script/general/compute/count.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_co_db diff --git a/tests/script/general/compute/diff.sim b/tests/script/general/compute/diff.sim index 88c2ecb09df9128499cef1426a53d5b922512164..bc303a9ca59152fb34836da6d37535d7e049d754 100644 --- a/tests/script/general/compute/diff.sim +++ b/tests/script/general/compute/diff.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_di_db diff --git a/tests/script/general/compute/diff2.sim b/tests/script/general/compute/diff2.sim index 14675c823f04693c76fcb151169b6dba738b8aaf..55fa1daa95516729fb37b2e541067fe7abda19e6 100644 --- a/tests/script/general/compute/diff2.sim +++ b/tests/script/general/compute/diff2.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_di_db diff --git a/tests/script/general/compute/first.sim b/tests/script/general/compute/first.sim index 01b82cce62adbc13420289878ea01ed63776bbce..fce334167b7b553bdd056b2372a2fae30f2af79b 100644 --- a/tests/script/general/compute/first.sim +++ b/tests/script/general/compute/first.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_fi_db diff --git a/tests/script/general/compute/interval.sim b/tests/script/general/compute/interval.sim index 8f9bf2ecb3013afe2bc11d5c851d8effbcf37686..c21003a6463d00d53a2b75bd402f5a2a905247a0 100644 --- a/tests/script/general/compute/interval.sim +++ b/tests/script/general/compute/interval.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_in_db diff --git a/tests/script/general/compute/last.sim b/tests/script/general/compute/last.sim index 88a6a268463ef005dd4c611e2b127c3a0d8f3bf0..9f20f8c5aa63416119e2148da575166602d0a226 100644 --- a/tests/script/general/compute/last.sim +++ b/tests/script/general/compute/last.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_la_db diff --git a/tests/script/general/compute/last_row.sim b/tests/script/general/compute/last_row.sim index 55d97fe53c75b8df34caed98f4bd57bc05a9711f..3b28b0baa54d179693db7be1e7386a8701b8ff7f 100644 --- a/tests/script/general/compute/last_row.sim +++ b/tests/script/general/compute/last_row.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_la_db diff --git a/tests/script/general/compute/leastsquare.sim b/tests/script/general/compute/leastsquare.sim index 5a28e74bffeed53e3c7202a2dc5d3bf2551ed94b..1c8af7fe7fdceb6666000e79bb89ee7c11ac3cac 100644 --- a/tests/script/general/compute/leastsquare.sim +++ b/tests/script/general/compute/leastsquare.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_le_db diff --git a/tests/script/general/compute/max.sim b/tests/script/general/compute/max.sim index 1029ba78cc595b70048f887c55a2393cb2b46e10..f9665a823db130a0eb5b4a7eb169a5900948c8a8 100644 --- a/tests/script/general/compute/max.sim +++ b/tests/script/general/compute/max.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_ma_db diff --git a/tests/script/general/compute/min.sim b/tests/script/general/compute/min.sim index d6a57f4f20d01a4ab1fc95ec902785363d1f004b..4a9904e06594f63f4de7e13c193a2e8561ccbf10 100644 --- a/tests/script/general/compute/min.sim +++ b/tests/script/general/compute/min.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mi_db diff --git a/tests/script/general/compute/null.sim b/tests/script/general/compute/null.sim index 47f7a3c1b96152e5a30b4712831146187b98abc7..cd00b7a69dae3c039794b19573bd25cbb436be1a 100644 --- a/tests/script/general/compute/null.sim +++ b/tests/script/general/compute/null.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = db diff --git a/tests/script/general/compute/percentile.sim b/tests/script/general/compute/percentile.sim index 0fb88e7a40224404c2889696d6e8ba34655d18ee..b0f4fff8d7e9d36d62fab8a587e359ad9ab4f5ec 100644 --- a/tests/script/general/compute/percentile.sim +++ b/tests/script/general/compute/percentile.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_pe_db diff --git a/tests/script/general/compute/stddev.sim b/tests/script/general/compute/stddev.sim index abe4025e961460803e33c63c7243a1112a1bfab5..772ec8386a8d61167e4a208eef4c9b8830893f9c 100644 --- a/tests/script/general/compute/stddev.sim +++ b/tests/script/general/compute/stddev.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_st_db diff --git a/tests/script/general/compute/sum.sim b/tests/script/general/compute/sum.sim index 9d42e766b61e4c137b7f738dcb8b27d1f5061413..8fad9927504221b0313892360ab30b5d1e8b59c9 100644 --- a/tests/script/general/compute/sum.sim +++ b/tests/script/general/compute/sum.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_su_db diff --git a/tests/script/general/compute/top.sim b/tests/script/general/compute/top.sim index e6bcd4a5cb0cfb928d8e322b80aa10d801fe76e5..1e9d302b7c9b8d4e8c690ff372d3c802fa2a8d7a 100644 --- a/tests/script/general/compute/top.sim +++ b/tests/script/general/compute/top.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_to_db diff --git a/tests/script/general/connection/connection.sim b/tests/script/general/connection/connection.sim index abebbacbd94f0a3fc88083f980da77f7b7df8b9b..1af6e1fda6989c8a9c6eed36e579f8325eee0ac6 100644 --- a/tests/script/general/connection/connection.sim +++ b/tests/script/general/connection/connection.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/general/connection/mqtt.sim b/tests/script/general/connection/mqtt.sim index 4b291f91ea5089018207b11b2016b234dcc2a883..c2c50ef17e1264d6f27f64417f1500bc83346012 100644 --- a/tests/script/general/connection/mqtt.sim +++ b/tests/script/general/connection/mqtt.sim @@ -10,7 +10,7 @@ system sh/cfg.sh -n dnode1 -c mqtt -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database mqttdb; sql create table mqttdb.devices(ts timestamp, value double) tags(name binary(32), model binary(32), serial binary(16), param binary(16), unit binary(16)); diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim index 1c3f543ffdb149d67cfaf1527a0e88f46f626913..170ba21c28b7ff4a79a8f2375dc1d36ba91e6120 100644 --- a/tests/script/general/db/alter_option.sim +++ b/tests/script/general/db/alter_option.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= create database @@ -78,23 +78,10 @@ if $data5_db != 1 then return -1 endi -sql alter database db quorum 2 -sql show databases -print quorum $data5_db -if $data5_db != 2 then - return -1 -endi +sql_error alter database db quorum 2 -sql alter database db quorum 3 -sql show databases -print quorum $data5_db -if $data5_db != 3 then - return -1 -endi +sql_error alter database db quorum 3 -sql alter database db quorum 3 -sql alter database db quorum 2 -sql alter database db quorum 1 sql_error alter database db quorum 0 sql_error alter database db quorum 4 sql_error alter database db quorum 5 @@ -222,7 +209,7 @@ sql alter database db wal 1 sql alter database db wal 2 sql alter database db wal 1 sql alter database db wal 2 -sql_error alter database db wal 0 +sql alter database db wal 0 sql_error alter database db wal 3 sql_error alter database db wal 4 sql_error alter database db wal -1 diff --git a/tests/script/general/db/alter_tables_d2.sim b/tests/script/general/db/alter_tables_d2.sim index cd3121057b2c8d02755e89c40e11ca7b9d52b4ce..f74f98d571e49090e2c25d2371a0e0c268c9a3ee 100644 --- a/tests/script/general/db/alter_tables_d2.sim +++ b/tests/script/general/db/alter_tables_d2.sim @@ -264,10 +264,10 @@ endi print ============================ step7 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql reset query cache sleep 1000 diff --git a/tests/script/general/db/alter_tables_v4.sim b/tests/script/general/db/alter_tables_v4.sim index b57b2c03208274949a901cf72c2d580133ca1ecd..10bb4e108bf7fc6c879e230565dad29599f15549 100644 --- a/tests/script/general/db/alter_tables_v4.sim +++ b/tests/script/general/db/alter_tables_v4.sim @@ -232,9 +232,9 @@ endi print ============================ step7 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql reset query cache sleep 1000 diff --git a/tests/script/general/db/alter_vgroups.sim b/tests/script/general/db/alter_vgroups.sim index 13928cf03301f1528db3a1fd9e5951b3115a6125..81ffc7d443ce6b21978d338bfeb01f00f89a17b1 100644 --- a/tests/script/general/db/alter_vgroups.sim +++ b/tests/script/general/db/alter_vgroups.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ step1 @@ -70,9 +70,9 @@ endi print ============================ step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 2 -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql create table db.t100 using db.st tags(0) sql create table db.t101 using db.st tags(1) @@ -132,9 +132,9 @@ print ============================ step5 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql create table db.t200 using db.st tags(0) sql create table db.t201 using db.st tags(1) diff --git a/tests/script/general/db/backup/keep.sim b/tests/script/general/db/backup/keep.sim index 29771fc97815c6781cfa3e44fbbfa81760859633..4d157b39857a639a83d24e1d8eb027492a4d40de 100644 --- a/tests/script/general/db/backup/keep.sim +++ b/tests/script/general/db/backup/keep.sim @@ -6,9 +6,9 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 @@ -23,7 +23,7 @@ sql connect print ========= start other dnodes sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 print ======== step1 create db sql create database keepdb replica 1 keep 30 days 7 @@ -50,9 +50,9 @@ endi print ======== step2 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql select * from tb print ===> rows $rows @@ -112,9 +112,9 @@ endi print ======== step5 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql select * from tb print ===> rows $rows @@ -153,9 +153,9 @@ endi print ======== step7 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +sleep 2000 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql select * from tb print ===> rows $rows diff --git a/tests/script/general/db/basic.sim b/tests/script/general/db/basic.sim index 1c4939256be82541f40e34d5c89ea4c1f67de288..684ce825fecb64ef580751a8df104fd4e1c7cf24 100644 --- a/tests/script/general/db/basic.sim +++ b/tests/script/general/db/basic.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/db/basic1.sim b/tests/script/general/db/basic1.sim index 302c5ac40992271e9ef757cb9704beb377a75d16..9ec1aabe98ba21af5927faec2e26aef1c3a14aeb 100644 --- a/tests/script/general/db/basic1.sim +++ b/tests/script/general/db/basic1.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database diff --git a/tests/script/general/db/basic2.sim b/tests/script/general/db/basic2.sim index afe5ee5d95402fac985b9c5673fb43e35a285f22..acd035bd744eb53b856c32beaf69445eb24e8315 100644 --- a/tests/script/general/db/basic2.sim +++ b/tests/script/general/db/basic2.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database d1 diff --git a/tests/script/general/db/basic3.sim b/tests/script/general/db/basic3.sim index 88f5bf64604d2f1f6bd11040de844e871808c566..fb64476696c7aba65d5f5f87c68857ed32ff24fc 100644 --- a/tests/script/general/db/basic3.sim +++ b/tests/script/general/db/basic3.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database d1 diff --git a/tests/script/general/db/basic4.sim b/tests/script/general/db/basic4.sim index a0a9aaa627fd643b2863b14a4cacb93f092104b0..ce6d352d12412c95d0883ca246e2603b36617bb8 100644 --- a/tests/script/general/db/basic4.sim +++ b/tests/script/general/db/basic4.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database d1 diff --git a/tests/script/general/db/basic5.sim b/tests/script/general/db/basic5.sim index 82b9bf9bf4dadf2c0215626d8905a64cc0962fe3..08c9db23323ba7d4257671d6f0a71c3dd133ad03 100644 --- a/tests/script/general/db/basic5.sim +++ b/tests/script/general/db/basic5.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database d1 diff --git a/tests/script/general/db/delete.sim b/tests/script/general/db/delete.sim index e4d40bf5da267985066f61383c6614dee78aaff1..4384044885e7c6a660079551b1190481ace3554e 100644 --- a/tests/script/general/db/delete.sim +++ b/tests/script/general/db/delete.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 @@ -44,7 +44,7 @@ endi print ======= step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start $x = 0 diff --git a/tests/script/general/db/delete_reuse1.sim b/tests/script/general/db/delete_reuse1.sim index feeb0152c1d8dc707a327f89ebf95e242b35294b..9d02e041c461000982079b86b5359a0b5d3107a3 100644 --- a/tests/script/general/db/delete_reuse1.sim +++ b/tests/script/general/db/delete_reuse1.sim @@ -5,10 +5,10 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 @@ -22,7 +22,7 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/db/delete_reuse2.sim b/tests/script/general/db/delete_reuse2.sim index 9053e8af011674162174a718c52d157b1e70984a..889c8f46f74783ff084e1dfaa06fb1fc9d900d5d 100644 --- a/tests/script/general/db/delete_reuse2.sim +++ b/tests/script/general/db/delete_reuse2.sim @@ -5,10 +5,10 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 @@ -21,9 +21,9 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes -sleep 2000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 + sql connect sql reset query cache @@ -65,7 +65,7 @@ endi print ======== step2 sql drop database d1 -sleep 1000 +sleep 500 sql insert into d1.t1 values(now, 2) -x step2 return -1 step2: @@ -73,7 +73,7 @@ step2: print ========= step3 sql create database db1 replica 1 sql reset query cache -sleep 1000 +sleep 500 sql create table db1.tb1 (ts timestamp, i int) sql insert into db1.tb1 values(now, 2) sql select * from db1.tb1 @@ -90,7 +90,7 @@ while $x < 20 sql use $db sql drop database $db - sleep 1000 + sleep 500 sql insert into $tb values(now, -1) -x step4 return -1 step4: @@ -100,7 +100,7 @@ while $x < 20 $tb = tb . $x sql reset query cache - sleep 1000 + sleep 500 sql create database $db replica 1 sql use $db diff --git a/tests/script/general/db/delete_reusevnode2.sim b/tests/script/general/db/delete_reusevnode2.sim index 0db2440b3bcd92a9fa252f387be22767ae1f7d7c..c05db6b65a43df2cec9cb410aef69ac23985307e 100644 --- a/tests/script/general/db/delete_reusevnode2.sim +++ b/tests/script/general/db/delete_reusevnode2.sim @@ -4,7 +4,7 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/db/delete_writing1.sim b/tests/script/general/db/delete_writing1.sim index 8b369b4e3d0fc918aad19d2295583a0e2c1fda59..010bd707434afe3fe85c2f8d0ecbe77e67d6b252 100644 --- a/tests/script/general/db/delete_writing1.sim +++ b/tests/script/general/db/delete_writing1.sim @@ -5,10 +5,10 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 @@ -43,7 +43,7 @@ while $x < 20 sql create database db sql create table db.tb (ts timestamp, i int) - sleep 3000 + sleep 2000 $x = $x + 1 endw diff --git a/tests/script/general/db/delete_writing2.sim b/tests/script/general/db/delete_writing2.sim index 3ea220cb8ce9ab2e30ab4c57652b3e35add56452..2d1b0c8d9cb4b4bfc8774436accf6b8ff90843da 100644 --- a/tests/script/general/db/delete_writing2.sim +++ b/tests/script/general/db/delete_writing2.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c wallevel -v 0 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db @@ -39,7 +39,7 @@ while $x < 10 sql create database db sql create table db.tb (ts timestamp, i int) - sleep 3000 + sleep 2000 $x = $x + 1 endw diff --git a/tests/script/general/db/dropdnodes.sim b/tests/script/general/db/dropdnodes.sim index c7bbdf73a4b6b6e1f236c7cc920a43def08bd013..8a46d5f9ce405cb84957900c37986cbede044464 100644 --- a/tests/script/general/db/dropdnodes.sim +++ b/tests/script/general/db/dropdnodes.sim @@ -11,7 +11,7 @@ print ========== prepare data system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 @@ -72,7 +72,7 @@ endi print ========== step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql drop dnode $hostname2 sleep 2000 diff --git a/tests/script/general/db/len.sim b/tests/script/general/db/len.sim index 30abcbe10010cd4a6bca8285c9d526133e9ea1f3..8c08d2b09afe713907f6e8f104874ee19b7a2fae 100644 --- a/tests/script/general/db/len.sim +++ b/tests/script/general/db/len.sim @@ -2,11 +2,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 diff --git a/tests/script/general/db/nosuchfile.sim b/tests/script/general/db/nosuchfile.sim index 98ac4ec012dc694357878a61ca0dbc11259f0a9e..69db8c0dc53ff39fff9840b3c86deef85e6d6dfa 100644 --- a/tests/script/general/db/nosuchfile.sim +++ b/tests/script/general/db/nosuchfile.sim @@ -6,7 +6,7 @@ system sh/cfg.sh -n dnode1 -c wallevel -v 2 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ========== step3 sql create database d1 @@ -20,7 +20,7 @@ sql insert into d1.t1 values(now+5s, 31) print ========== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 print ========== step5 sql select * from d1.t1 order by t desc diff --git a/tests/script/general/db/repeat.sim b/tests/script/general/db/repeat.sim index aaa103234d1e3206adce7511e4ac5e105b2dc6c4..b3bbca2d19012e49c3c4210dbbb3b94256cfe278 100644 --- a/tests/script/general/db/repeat.sim +++ b/tests/script/general/db/repeat.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -21,13 +21,13 @@ sql create table d3.t1(ts timestamp, i int) sql create database d4 sql create table d4.t1(ts timestamp, i int) -sleep 3000 +sleep 2000 sql drop database d1 sql drop database d2 sql drop database d3 sql drop database d4 -sleep 3000 +sleep 2000 sql create database d5 sql create table d5.t1(ts timestamp, i int) @@ -41,14 +41,14 @@ sql create table d7.t1(ts timestamp, i int) sql create database d8 sql create table d8.t1(ts timestamp, i int) -sleep 3000 +sleep 2000 sql drop database d5 sql drop database d6 sql drop database d7 sql drop database d8 -sleep 3000 +sleep 2000 sql create database d9; sql create table d9.t1(ts timestamp, i int) diff --git a/tests/script/general/db/show_create_db.sim b/tests/script/general/db/show_create_db.sim index baa7b253e12154c5d320dd6d8411e223ce1cd4e2..f2e8011a0e789792a1a9a345e7f59eb3a8d4a0e7 100644 --- a/tests/script/general/db/show_create_db.sim +++ b/tests/script/general/db/show_create_db.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step2 diff --git a/tests/script/general/db/show_create_table.sim b/tests/script/general/db/show_create_table.sim index 83386387093a7c4f778bb5696eea6019fd4691fa..5f2ae61343f51e52a39b11ebce75de2f2c305cad 100644 --- a/tests/script/general/db/show_create_table.sim +++ b/tests/script/general/db/show_create_table.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ===============create three type table diff --git a/tests/script/general/db/tables.sim b/tests/script/general/db/tables.sim index d700bf80689a8d0413b59e13382fdf213ee9c4ec..b32fae25e0a12084e473cd82f482501eb3d99716 100644 --- a/tests/script/general/db/tables.sim +++ b/tests/script/general/db/tables.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step2 diff --git a/tests/script/general/db/topic1.sim b/tests/script/general/db/topic1.sim new file mode 100644 index 0000000000000000000000000000000000000000..42613405afda7580003f58ae82f950880d60de62 --- /dev/null +++ b/tests/script/general/db/topic1.sim @@ -0,0 +1,884 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 100 +system sh/cfg.sh -n dnode1 -c partitions -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +print ====step1 create with default para +sql create topic t1; +sql use t1; + +sql show topics; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +if $data02 != 4 then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +#tables +if $data02 < 1 then + return -1 +endi +#numofvgroups +if $data03 < 1 then + return -1 +endi + +sql show t1.vgroups; +if $rows < 1 then + return -1 +endi + +sql show t1.stables; +if $rows != 1 then + return -1 +endi +if $data04 < 1 then + return -1 +endi + +sql show t1.tables; +if $rows < 1 then + return -1 +endi + +sql drop topic t1 +sql show topics; +if $rows != 0 then + return -1 +endi +sql show databases; +if $rows != 0 then + return -1 +endi + +sql_error use t1; +sql_error show t1.vgroups; +sql_error show t1.stables; +sql_error show t1.tables; + +print ====step2 create with giving para +sql create topic t1 partitions 6; + +sql show topics; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +if $data02 != 6 then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +#tables +if $data02 != 6 then + return -1 +endi +#numofvgroups +if $data03 != 6 then + return -1 +endi + +sql show t1.vgroups; +if $rows != 6 then + return -1 +endi + +sql show t1.stables; +if $rows != 1 then + return -1 +endi +if $data00 != ps then + return -1 +endi +if $data04 != 6 then + return -1 +endi + +sql show t1.tables; +if $rows != 6 then + return -1 +endi + +sql describe t1.ps; +if $data00 != off then + return -1 +endi +if $data10 != ts then + return -1 +endi +if $data20 != content then + return -1 +endi +if $data30 != pid then + return -1 +endi + +sql describe t1.p1; +if $data00 != off then + return -1 +endi +if $data10 != ts then + return -1 +endi +if $data20 != content then + return -1 +endi +if $data30 != pid then + return -1 +endi + +sql drop topic t1 +sql show topics; +if $rows != 0 then + return -1 +endi + +sql show databases; +if $rows != 0 then + return -1 +endi + +sql_error show t1.vgroups; +sql_error show t1.stables; +sql_error show t1.tables; + +sql_error create topic t1 partitions -1; +#sql_error create topic t1 partitions 0; +sql_error create topic t1 partitions 10001; + +print =============step3 create with db para +sql create topic db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 +sql show databases +if $data00 != db then + return -1 +endi +if $data02 != 4 then + return -1 +endi +if $data03 != 4 then + return -1 +endi +if $data04 != 1 then + return -1 +endi +if $data06 != 10 then + return -1 +endi +if $data07 != 20,20,20 then + return -1 +endi +if $data08 != 2 then + return -1 +endi +if $data09 != 4 then + return -1 +endi +sql drop topic db; + +sql create topic db cache 2 blocks 4 days 10 keep 20 minRows 300 maxRows 400 ctime 120 precision 'ms' comp 2 wal 1 replica 1 partitions 7 +sql show databases +if $data00 != db then + return -1 +endi +if $data02 != 7 then + return -1 +endi +if $data03 != 7 then + return -1 +endi +if $data04 != 1 then + return -1 +endi +if $data06 != 10 then + return -1 +endi +if $data07 != 20,20,20 then + return -1 +endi +if $data08 != 2 then + return -1 +endi +if $data09 != 4 then + return -1 +endi + +sql show topics; +if $rows != 1 then + return -1 +endi +if $data00 != db then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi +if $data00 != db then + return -1 +endi +#tables +if $data02 != 7 then + return -1 +endi +#numofvgroups +sql show db.vgroups; +if $rows != 7 then + return -1 +endi +sql show db.stables; +if $rows != 1 then + return -1 +endi +sql show db.tables; +if $rows != 7 then + return -1 +endi + +print ============== step name +sql_error alter database db name d1 +sql_error alter database db name d2 +sql_error alter topic db name d1 +sql_error alter topic db name d2 + +print ============== step ntables +sql_error alter database db ntables -1 +sql_error alter database db ntables 0 +sql_error alter database db ntables 1 +sql_error alter database db ntables 10 +sql_error alter topic db ntables -1 +sql_error alter topic db ntables 0 +sql_error alter topic db ntables 1 +sql_error alter topic db ntables 10 + +print ============== step vgroups +sql_error alter database db vgroups -1 +sql_error alter database db vgroups 0 +sql_error alter database db vgroups 1 +sql_error alter database db vgroups 10 +sql_error alter topic db vgroups -1 +sql_error alter topic db vgroups 0 +sql_error alter topic db vgroups 1 +sql_error alter topic db vgroups 10 + +print ============== step replica +sql_error alter database db replica 2 +sql_error alter database db replica 3 +sql_error alter database db replica 0 +sql_error alter topic db replica 2 +sql_error alter topic db replica 3 +sql_error alter topic db replica 0 + +sql alter database db replica 1 +sql show databases +print replica $data4_db +if $data4_db != 1 then + return -1 +endi + +sql show topics +if $rows != 1 then + return -1 +endi + +print ============== step quorum +sql show databases +print quorum $data5_db +if $data5_db != 1 then + return -1 +endi + +sql_error alter topic db quorum 1 +sql alter database db quorum 1 +sql show databases +print quorum $data5_db +if $data5_db != 1 then + return -1 +endi + +sql_error alter database db quorum 2 +sql_error alter database db quorum 3 +sql_error alter topic db quorum 2 +sql_error alter topic db quorum 3 + +sql_error alter database db quorum 0 +sql_error alter database db quorum 4 +sql_error alter database db quorum 5 +sql_error alter database db quorum -1 +sql_error alter topic db quorum 0 +sql_error alter topic db quorum 4 +sql_error alter topic db quorum 5 +sql_error alter topic db quorum -1 + +print ============== step days +sql_error alter database db days 0 +sql_error alter database db days 1 +sql_error alter database db days 2 +sql_error alter database db days 10 +sql_error alter database db days 50 +sql_error alter database db days 100 +sql_error alter topic db days 0 +sql_error alter topic db days 1 +sql_error alter topic db days 2 +sql_error alter topic db days 10 +sql_error alter topic db days 50 +sql_error alter topic db days 100 + +print ============== step keep +sql show databases +print keep $data7_db +if $data7_db != 20,20,20 then + return -1 +endi + +sql_error topic db keep 20 +sql alter database db keep 20 +sql show databases +print keep $data7_db +if $data7_db != 20,20,20 then + return -1 +endi + +sql_error topic db keep 30 +sql alter database db keep 30 +sql show databases +print keep $data7_db +if $data7_db != 20,20,30 then + return -1 +endi + +sql_error alter topic db keep 40 +sql alter database db keep 40 +sql show databases +print keep $data7_db +if $data7_db != 20,20,40 then + return -1 +endi + +sql alter database db keep 40 +sql alter database db keep 30 +sql alter database db keep 20 +sql_error alter database db keep 10 +sql_error alter database db keep 9 +sql_error alter database db keep 1 +sql alter database db keep 0 +sql alter database db keep -1 +sql_error alter database db keep 365001 + +sql_error alter topic db keep 40 +sql_error alter topic db keep 30 +sql_error alter topic db keep 20 +sql_error alter topic db keep 10 +sql_error alter topic db keep 9 +sql_error alter topic db keep 1 +sql_error alter topic db keep 0 +sql_error alter topic db keep -1 +sql_error alter topic db keep 365001 + +print ============== step cache +sql_error alter database db cache 60 +sql_error alter database db cache 50 +sql_error alter database db cache 20 +sql_error alter database db cache 3 +sql_error alter database db cache 129 +sql_error alter database db cache 300 +sql_error alter database db cache 0 +sql_error alter database db cache -1 + +sql_error alter topic db cache 60 +sql_error alter topic db cache 50 +sql_error alter topic db cache 20 +sql_error alter topic db cache 3 +sql_error alter topic db cache 129 +sql_error alter topic db cache 300 +sql_error alter topic db cache 0 +sql_error alter topic db cache -1 + +print ============== step blocks +sql show databases +print blocks $data9_db +if $data9_db != 4 then + return -1 +endi + +sql_error alter topic db blocks 10 +sql alter database db blocks 10 +sql show databases +print blocks $data9_db +if $data9_db != 10 then + return -1 +endi + +sql_error alter topic db blocks 20 +sql alter database db blocks 20 +sql show databases +print blocks $data9_db +if $data9_db != 20 then + return -1 +endi + +sql_error alter topic db blocks 20 +sql alter database db blocks 30 +sql show databases +print blocks $data9_db +if $data9_db != 30 then + return -1 +endi + +sql alter database db blocks 40 +sql alter database db blocks 30 +sql alter database db blocks 20 +sql alter database db blocks 10 +sql_error alter database db blocks 2 +sql_error alter database db blocks 1 +sql alter database db blocks 0 +sql_error alter database db blocks -1 +sql_error alter database db blocks 10001 + +sql_error alter topic db blocks 40 +sql_error alter topic db blocks 30 +sql_error alter topic db blocks 20 +sql_error alter topic db blocks 10 +sql_error alter topic db blocks 2 +sql_error alter topic db blocks 1 +sql_error alter topic db blocks 0 +sql_error alter topic db blocks -1 +sql_error alter topic db blocks 10001 + +print ============== step minrows +sql_error alter database db minrows 1 +sql_error alter database db minrows 100 +sql_error alter database db minrows 1000 + +sql_error alter topic db minrows 1 +sql_error alter topic db minrows 100 +sql_error alter topic db minrows 1000 + +print ============== step maxrows +sql_error alter database db maxrows 1 +sql_error alter database db maxrows 100 +sql_error alter database db maxrows 1000 + +sql_error alter topic db maxrows 1 +sql_error alter topic db maxrows 100 +sql_error alter topic db maxrows 1000 + +print ============== step wallevel +sql show databases +print wallevel $data12_db +if $data12_db != 1 then + return -1 +endi + +sql_error alter topic db wal 1 +sql alter database db wal 1 +sql show databases +print wal $data12_db +if $data12_db != 1 then + return -1 +endi + +sql alter database db wal 1 +sql alter database db wal 2 +sql alter database db wal 1 +sql alter database db wal 2 +sql alter database db wal 0 +sql_error alter database db wal 3 +sql_error alter database db wal 4 +sql_error alter database db wal -1 +sql_error alter database db wal 1000 + +sql_error alter topic db wal 1 +sql_error alter topic db wal 2 +sql_error alter topic db wal 1 +sql_error alter topic db wal 2 +sql_error alter topic db wal 0 +sql_error alter topic db wal 3 +sql_error alter topic db wal 4 +sql_error alter topic db wal -1 +sql_error alter topic db wal 1000 + +print ============== step fsync +sql alter database db fsync 0 +sql alter database db fsync 1 +sql alter database db fsync 3600 +sql alter database db fsync 18000 +sql alter database db fsync 180000 +sql_error alter database db fsync 180001 +sql_error alter database db fsync -1 + +sql_error alter topic db fsync 0 +sql_error alter topic db fsync 1 +sql_error alter topic db fsync 3600 +sql_error alter topic db fsync 18000 +sql_error alter topic db fsync 180000 +sql_error alter topic db fsync 180001 +sql_error alter topic db fsync -1 + +print ============== step comp +sql show databases +print comp $data14_db +if $data14_db != 2 then + return -1 +endi + +sql_error alter topic db comp 1 +sql alter database db comp 1 +sql show databases +print comp $data14_db +if $data14_db != 1 then + return -1 +endi + +sql_error alter topic db comp 2 +sql alter database db comp 2 +sql show databases +print comp $data14_db +if $data14_db != 2 then + return -1 +endi + +sql_error alter topic db comp 0 +sql alter database db comp 0 +sql show databases +print comp $data14_db +if $data14_db != 0 then + return -1 +endi + +sql_error alter database db comp 3 +sql_error alter database db comp 4 +sql_error alter database db comp 5 +sql_error alter database db comp -1 + +sql_error alter topic db comp 3 +sql_error alter topic db comp 4 +sql_error alter topic db comp 5 +sql_error alter topic db comp -1 + +print ============== step precision +sql_error alter database db prec 'us' +sql_error alter topic db prec 'us' + +print ============== step status +sql_error alter database db status 'delete' +sql_error alter topic db status 'delete' + +print ============== step drop +sql drop database db +sql show topics; +if $rows != 0 then + return -1 +endi + +sql show databases; +if $rows != 0 then + return -1 +endi + +print ============== step db1 +sql create database d1 +sql_error alter database d1 partitions 2 +sql_error alter topic d1 partitions 2 + +sql show topics; +if $rows != 0 then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi + +sql alter database d1 fsync 0 +sql show topics; +if $rows != 0 then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi + +sql drop database d1 +sql show topics; +if $rows != 0 then + return -1 +endi + +sql show databases; +if $rows != 0 then + return -1 +endi + +print ============== step db2 +sql create topic d1 +sql show topics; +if $rows != 1 then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi + +sql alter database d1 fsync 0 +sql show topics; +if $rows != 1 then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi + +sql drop database d1 +sql show topics; +if $rows != 0 then + return -1 +endi + +sql show databases; +if $rows != 0 then + return -1 +endi + +print ============== step db3 +sql create topic d1 +sql show topics; +if $rows != 1 then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi + +sql alter topic d1 partitions 2 +sql show topics; +if $rows != 1 then + return -1 +endi + +sql show databases; +if $rows != 1 then + return -1 +endi + +sql drop database d1 +sql show topics; +if $rows != 0 then + return -1 +endi + +sql show databases; +if $rows != 0 then + return -1 +endi + +print ============== step partitions +sql create topic t1 partitions 5 + +sql_error alter database t1 partitions -1 +sql_error alter database t1 partitions 0 +sql_error alter database t1 partitions 1 +sql_error alter database t1 partitions 2 +sql_error alter database t1 partitions 3 +sql_error alter database t1 partitions 100 +sql_error alter database t1 partitions 1000 +sql_error alter database t1 partitions 10000 + +sql_error alter topic t1 partitions -1 +#sql_error alter topic t1 partitions 0 +sql_error alter database t1 partitions 10000 + +sql alter topic t1 partitions 1 +sql show topics; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +sql show databases; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +#tables +if $data02 != 1 then + return -1 +endi + +sql show t1.stables; +if $rows != 1 then + return -1 +endi +sql show t1.tables; +if $rows != 1 then + return -1 +endi + +sql alter topic t1 partitions 2 +sql show topics; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +sql show databases; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +#tables +if $data02 != 2 then + return -1 +endi + + +sql show t1.stables; +if $rows != 1 then + return -1 +endi +sql show t1.tables; +if $rows != 2 then + return -1 +endi + +sql alter topic t1 partitions 3 +sql show topics; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +sql show databases; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +#tables +if $data02 != 3 then + return -1 +endi + +sql show t1.stables; +if $rows != 1 then + return -1 +endi +sql show t1.tables; +if $rows != 3 then + return -1 +endi + +sql alter topic t1 partitions 10 +sql show topics; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +sql show databases; +if $rows != 1 then + return -1 +endi +if $data00 != t1 then + return -1 +endi +#tables +if $data02 != 10 then + return -1 +endi +#numofvgroups +sql show t1.vgroups; +if $rows != 10 then + return -1 +endi +sql show t1.stables; +if $rows != 1 then + return -1 +endi +sql show t1.tables; +if $rows != 10 then + return -1 +endi + +sql drop topic t1 + +print ============== create same name topic +sql create database d2 +sql create topic t2 + +sql_error create topic d2 +sql_error create topic if not exists d2 +sql_error create topic t2 +sql create topic if not exists t2 +sql_error create topic t2 partitions 5; +sql_error create topic t2 partitions 6; +sql_error create topic t2 partitions 3; + +sql_error alter topic t3 partitions 1 +sql_error alter topic d2 partitions 1 +#sql_error alter topic t2 partitions 0 +sql_error alter topic t2 partitions 10000 + +sql_error drop topic d2 +sql_error drop topic d3 +sql drop database d2 +sql drop database t2 + +print ============== create partitons 0 +sql create topic t2 partitions 0 +sql show t2.stables; +if $rows != 0 then + return -1 +endi +sql show t2.tables; +if $rows != 0 then + return -1 +endi + +sql drop topic t2 +sql_error drop topic abc +sql drop topic if exists abc; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/db/topic2.sim b/tests/script/general/db/topic2.sim new file mode 100644 index 0000000000000000000000000000000000000000..f933f5eee4dd9d0ec4784ba1e5beec4ccfc9a029 --- /dev/null +++ b/tests/script/general/db/topic2.sim @@ -0,0 +1,321 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +print ==== step1 +sql create topic t1 partitions 2; +sql show t1.tables +if $rows != 2 then + return -1 +endi +sql show t1.vgroups +if $rows != 2 then + return -1 +endi + +sql insert into t1.p1 values(1, now, '1'); +sql insert into t1.p1 values(1, now, '2'); +sql insert into t1.p1 values(1, now, '3'); +sql insert into t1.p1 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p1 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p1 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql insert into t1.p2 values(1, now, '1'); +sql insert into t1.p2 values(1, now, '2'); +sql insert into t1.p2 values(1, now, '3'); +sql insert into t1.p2 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p2 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p2 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql_error insert into t1.p3 values(1, now, '1'); +sql_error insert into t1.p3 values(1, now, '2'); +sql_error insert into t1.p3 values(1, now, '3'); +sql_error insert into t1.p3 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql_error insert into t1.p3 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql_error insert into t1.p3 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql select * from t1.p1 order by off asc +if $rows != 33 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +sql select * from t1.p2 order by off asc +if $rows != 33 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +print ==== step2 +sql alter topic t1 partitions 4; +sql show t1.tables +if $rows != 4 then + return -1 +endi +sql show t1.vgroups +if $rows != 4 then + return -1 +endi + +sql insert into t1.p1 values(1, now, '1'); +sql insert into t1.p1 values(1, now, '2'); +sql insert into t1.p1 values(1, now, '3'); +sql insert into t1.p1 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p1 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p1 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql insert into t1.p2 values(1, now, '1'); +sql insert into t1.p2 values(1, now, '2'); +sql insert into t1.p2 values(1, now, '3'); +sql insert into t1.p2 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p2 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p2 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql insert into t1.p3 values(1, now, '1'); +sql insert into t1.p3 values(1, now, '2'); +sql insert into t1.p3 values(1, now, '3'); +sql insert into t1.p3 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p3 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p3 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql insert into t1.p4 values(1, now, '1'); +sql insert into t1.p4 values(1, now, '2'); +sql insert into t1.p4 values(1, now, '3'); +sql insert into t1.p4 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p4 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p4 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql_error insert into t1.p5 values(1, now, '1'); +sql_error insert into t1.p5 values(1, now, '2'); +sql_error insert into t1.p5 values(1, now, '3'); +sql_error insert into t1.p5 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql_error insert into t1.p5 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql_error insert into t1.p5 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql select * from t1.p1 order by off asc +if $rows != 66 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +sql select * from t1.p2 order by off asc +if $rows != 66 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +sql select * from t1.p3 order by off asc +if $rows != 33 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +sql select * from t1.p4 order by off asc +if $rows != 33 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +print ==== step3 +sql alter topic t1 partitions 1; +sql show t1.tables +if $rows != 1 then + return -1 +endi +sql show t1.vgroups +if $rows != 1 then + return -1 +endi + +sql insert into t1.p1 values(1, now, '1'); +sql insert into t1.p1 values(1, now, '2'); +sql insert into t1.p1 values(1, now, '3'); +sql insert into t1.p1 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p1 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p1 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql_error insert into t1.p2 values(1, now, '1'); +sql_error insert into t1.p2 values(1, now, '2'); +sql_error insert into t1.p2 values(1, now, '3'); +sql_error insert into t1.p2 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql_error insert into t1.p2 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql_error insert into t1.p2 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql_error insert into t1.p3 values(1, now, '1'); +sql_error insert into t1.p3 values(1, now, '2'); +sql_error insert into t1.p3 values(1, now, '3'); +sql_error insert into t1.p3 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql_error insert into t1.p3 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql_error insert into t1.p3 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql_error insert into t1.p4 values(1, now, '1'); +sql_error insert into t1.p4 values(1, now, '2'); +sql_error insert into t1.p4 values(1, now, '3'); +sql_error insert into t1.p4 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql_error insert into t1.p4 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql_error insert into t1.p4 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql_error insert into t1.p5 values(1, now, '1'); +sql_error insert into t1.p5 values(1, now, '2'); +sql_error insert into t1.p5 values(1, now, '3'); +sql_error insert into t1.p5 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql_error insert into t1.p5 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql_error insert into t1.p5 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql select * from t1.p1 order by off asc +if $rows != 99 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +sql_error select * from t1.p2 order by off asc +sql_error select * from t1.p3 order by off asc +sql_error select * from t1.p4 order by off asc + +print ==== step4 +sql alter topic t1 partitions 3; +sql show t1.tables +if $rows != 3 then + return -1 +endi +sql show t1.vgroups +if $rows != 3 then + return -1 +endi + +sql insert into t1.p1 values(1, now, '1'); +sql insert into t1.p1 values(1, now, '2'); +sql insert into t1.p1 values(1, now, '3'); +sql insert into t1.p1 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p1 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p1 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql insert into t1.p2 values(1, now, '1'); +sql insert into t1.p2 values(1, now, '2'); +sql insert into t1.p2 values(1, now, '3'); +sql insert into t1.p2 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p2 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p2 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql insert into t1.p3 values(1, now, '1'); +sql insert into t1.p3 values(1, now, '2'); +sql insert into t1.p3 values(1, now, '3'); +sql insert into t1.p3 values(1, now, '4')(2, now, '5')(3, now, '6')(4, now, '7')(5, now, '8')(6, now, '9'); +sql insert into t1.p3 values(1, now, '10')(2, now, '11')(3, now, '12')(4, now, '13')(5, now, '14')(6, now, '15'); +sql insert into t1.p3 values(1, now, '16')(2, now,'17')(3, now,'18')(4, now,'19')(5, now,'20')(6, now,'21')(7, now,'22')(8, now,'23')(9, now,'24')(10, now,'25')(11, now,'26')(12, now,'27')(13, now,'28')(14, now,'29')(15, now,'30')(16, now,'31')(17, now,'32')(18, now,'33'); + +sql_error insert into t1.p4 values(1, now, '1'); +sql_error insert into t1.p5 values(1, now, '1'); +sql_error insert into t1.p6 values(1, now, '1'); +sql_error select * from t1.p4 order by off asc +sql_error select * from t1.p5 order by off asc +sql_error select * from t1.p6 order by off asc + +sql select * from t1.p1 order by off asc +if $rows != 132 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +sql select * from t1.p2 order by off asc +if $rows != 33 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +sql select * from t1.p3 order by off asc +if $rows != 33 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data12 != 2 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +sql select * from t1.ps order by off asc +if $rows != 198 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/db/vnodes.sim b/tests/script/general/db/vnodes.sim index b01e94206fa33978b359b6c40576284f69b74d87..96f68f16322a678ad6a6002a7f4fa6b092254616 100644 --- a/tests/script/general/db/vnodes.sim +++ b/tests/script/general/db/vnodes.sim @@ -5,7 +5,7 @@ $maxTables = 4 $totalRows = $totalVnodes * $maxTables system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v $maxTables system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v $totalVnodes system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 100000 @@ -15,7 +15,7 @@ system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 100000 print ========== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db blocks 3 cache 1 sql use db @@ -44,4 +44,4 @@ if $data00 != $totalRows then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/field/2.sim b/tests/script/general/field/2.sim index 28506b7cb82456ce424be5052a1f41269dcd56a3..dc39e5ad602276e52bda3cce0e823a3deb3f124d 100644 --- a/tests/script/general/field/2.sim +++ b/tests/script/general/field/2.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/3.sim b/tests/script/general/field/3.sim index 453f4968bfe1be218886bf958aaab44a287ba914..b45e3a005b4af41494ef24cf93b130d2b41c8dc1 100644 --- a/tests/script/general/field/3.sim +++ b/tests/script/general/field/3.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/4.sim b/tests/script/general/field/4.sim index 243424724fa8a487f8b8ea5394014c5913b097de..e219be87783cfea2f071db481a1f8eb457f20f5e 100644 --- a/tests/script/general/field/4.sim +++ b/tests/script/general/field/4.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/5.sim b/tests/script/general/field/5.sim index ca1543b54bf83ee40b077e2542e43e7e5c8e4c03..e02bbd122ff00a0f727ff8706e4a5cc3cf28a57b 100644 --- a/tests/script/general/field/5.sim +++ b/tests/script/general/field/5.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/6.sim b/tests/script/general/field/6.sim index 23223e5c1502105b5df13f9d9cad5ad12b08944c..a852230cea7fd53f1fbc8e6cd44a67ba137ab3b3 100644 --- a/tests/script/general/field/6.sim +++ b/tests/script/general/field/6.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/bigint.sim b/tests/script/general/field/bigint.sim index 10060f7422f50ccbb36b118d9dbfbc7729059e3f..3bb120c6410a2ea2e9c671bb4aca56fc8014124f 100644 --- a/tests/script/general/field/bigint.sim +++ b/tests/script/general/field/bigint.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/binary.sim b/tests/script/general/field/binary.sim index b51f023efe0bb768f90fc2f9e682363d208d0410..d601750b0d96b94bd60751de62c7fd139e1882be 100644 --- a/tests/script/general/field/binary.sim +++ b/tests/script/general/field/binary.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/bool.sim b/tests/script/general/field/bool.sim index 4528f79bc79ddb9fb6c2eeeb54c13384eec7df99..796ed4e0aaa4c1b0b61462ef95ffaf98a671f2ed 100644 --- a/tests/script/general/field/bool.sim +++ b/tests/script/general/field/bool.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/double.sim b/tests/script/general/field/double.sim index 40650cb9bdbc3908892a6e6ee1958fa62c90d6bb..ef86585e5f69fdd63c417ef823a57f3502436e40 100644 --- a/tests/script/general/field/double.sim +++ b/tests/script/general/field/double.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/float.sim b/tests/script/general/field/float.sim index 0ead9fb48a424e5d2c5d2ff3b69a6ecc21fbf540..a01bcbdd4ca2a7a7b7009ded641cd88357fbcc07 100644 --- a/tests/script/general/field/float.sim +++ b/tests/script/general/field/float.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/int.sim b/tests/script/general/field/int.sim index a095dc2176ad75c0123c7dcc9a1e0a5a1c75d75f..c04fe5d2b1d4cc2d43de0111d3e22dcd177fa8fc 100644 --- a/tests/script/general/field/int.sim +++ b/tests/script/general/field/int.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/single.sim b/tests/script/general/field/single.sim index 8540608e96aacfe50dc346a0bf4217f8c38db7b7..0cfb92aad5c795e14a304146419a97157176c3ac 100644 --- a/tests/script/general/field/single.sim +++ b/tests/script/general/field/single.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/smallint.sim b/tests/script/general/field/smallint.sim index 578de4ea442b87bea61f24db8c24b71efe0e2621..1d5566812efcd8e59ccc339e53577246c61c7a63 100644 --- a/tests/script/general/field/smallint.sim +++ b/tests/script/general/field/smallint.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/field/tinyint.sim b/tests/script/general/field/tinyint.sim index f132b427025d4c706cdcf09b4ede678e36fbaaf8..f10e3d293a9d847f4f71a585603598a69ddc5e1c 100644 --- a/tests/script/general/field/tinyint.sim +++ b/tests/script/general/field/tinyint.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/http/autocreate.sim b/tests/script/general/http/autocreate.sim index 98d64ab839df11d17bd6d9036a53fcc087b69ea8..39af990b50d18fbcb6e7972d2adb67a1eb9900e3 100644 --- a/tests/script/general/http/autocreate.sim +++ b/tests/script/general/http/autocreate.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/chunked.sim b/tests/script/general/http/chunked.sim index 6592c761c6ccaea94f385ac72982115c775f01d7..c5855e5d2953ca36d95cd84a5590c47ef693d406 100644 --- a/tests/script/general/http/chunked.sim +++ b/tests/script/general/http/chunked.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c maxSQLLength -v 340032 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/grafana.sim b/tests/script/general/http/grafana.sim index c7866e5f4ce950292f17a1b4d0f8a49838fc18a1..414b859bd3dcaa78ab7d814afd660c9894857cc3 100644 --- a/tests/script/general/http/grafana.sim +++ b/tests/script/general/http/grafana.sim @@ -1,13 +1,13 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c http -v 1 #system sh/cfg.sh -n dnode1 -c adminRowLimit -v 10 system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -84,7 +84,7 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/grafana/root/1/login print 6-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi @@ -94,7 +94,7 @@ if $system_content != @{"status":"error","code":4387,"desc":"invalid format of A return -1 endi -sleep 3000 +sleep 2000 system_content curl 127.0.0.1:7111/grafana/login/root/taosdata print 8-> $system_content if $system_content != @{"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"}@ then diff --git a/tests/script/general/http/grafana_bug.sim b/tests/script/general/http/grafana_bug.sim index 43c52ba75ff72f8567eebaec741f3edee3206d85..e66cd29deacc5a02d24568c236ffed79e114eb25 100644 --- a/tests/script/general/http/grafana_bug.sim +++ b/tests/script/general/http/grafana_bug.sim @@ -1,14 +1,14 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 #system sh/cfg.sh -n dnode1 -c adminRowLimit -v 10 system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ============================ dnode1 start diff --git a/tests/script/general/http/gzip.sim b/tests/script/general/http/gzip.sim index 9c77567abb9874459a2363a93cb62cb57cf02b1a..ce358d84a177b79f8333c2bb4ce370b79b4e0eca 100644 --- a/tests/script/general/http/gzip.sim +++ b/tests/script/general/http/gzip.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c maxSQLLength -v 340032 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/prepare.sim b/tests/script/general/http/prepare.sim index 0bcb42ad41baa48544e0febba6ccf693f3efb85b..4bf6b6119833563f85ee8f8de1a2393bf07a2e71 100644 --- a/tests/script/general/http/prepare.sim +++ b/tests/script/general/http/prepare.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/restful.sim b/tests/script/general/http/restful.sim index 7d1169ca27eac99e740aba3fb8c1f6483791a85b..6ebf5644e7a99593ebac0302c3dcdbc0a27fc8c1 100644 --- a/tests/script/general/http/restful.sim +++ b/tests/script/general/http/restful.sim @@ -1,12 +1,10 @@ system sh/stop_dnodes.sh -sleep 3000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql connect print ============================ dnode1 start @@ -41,14 +39,14 @@ print =============== step3 - query data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then +if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10}@ then return -1 endi print =============== step4 - insert data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.table_rest values('2017-12-25 21:28:51.022', 11)" 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi @@ -56,7 +54,7 @@ print =============== step5 - query data system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select * from d1.table_rest' 127.0.0.1:7111/rest/sql print curl 127.0.0.1:7111/rest/sql -----> $system_content -if $system_content != @{"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then +if $system_content != @{"status":"succ","head":["ts","i"],"column_meta":[["ts",9,8],["i",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then return -1 endi @@ -81,4 +79,4 @@ if $system_content != @{"status":"error","code":3,"desc":"Authentication failure return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index 7e12d30ac91eb49025fbebed05336e15cc6900ac..8094a943f507a702bb56a7d026da672134188b83 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/exec.sh -n dnode1 -s start -#sleep 3000 +#sleep 2000 sql connect print ============================ dnode1 start @@ -15,6 +15,7 @@ print =============== step1 - login system_content curl 127.0.0.1:7111/rest/ print 1-> $system_content if $system_content != @{"status":"error","code":4357,"desc":"no auth info input"}@ then + print $system_content return -1 endi @@ -56,19 +57,25 @@ if $system_content != @{"status":"error","code":3,"desc":"Authentication failure endi #8 -system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 +system_content curl -H 'Authorization: Beare eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 8-> $system_content if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then return -1 endi +system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 +print 8-> $system_content +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then + return -1 +endi + system_content curl -H 'Authorization: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' -d 'show databases' 127.0.0.1:7111/rest/login/root/1 print 9-> $system_content if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi -sleep 3000 +sleep 2000 system_content curl 127.0.0.1:7111/rest/login/root/taosdata/ print 10-> $system_content @@ -81,13 +88,13 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep0,keep1,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep0,keep1,keep(D)",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[],"rows":0}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d1' 127.0.0.1:7111/rest/sql print 12-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi @@ -153,26 +160,26 @@ endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d1.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql print 22-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 23-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[],"rows":0}@ then return -1 endi #24 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d1.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql print 24-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 25-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then return -1 endi @@ -201,32 +208,32 @@ system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl #27 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d1.t1 ' 127.0.0.1:7111/rest/sql print 27-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10],["2017-12-25 21:28:51.022",11]],"rows":11}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'create database d2' 127.0.0.1:7111/rest/sql print 28-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' create table d2.t1 (ts timestamp, speed int)' 127.0.0.1:7111/rest/sql print 29-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[0]],"rows":0}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[0]],"rows":1}@ then return -1 endi #30 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d "insert into d2.t1 values('2017-12-25 21:28:41.022', 1)" 127.0.0.1:7111/rest/sql print 30-> $system_content -if $system_content != @{"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["affected_rows"],"column_meta":[["affected_rows",4,4]],"data":[[1]],"rows":1}@ then return -1 endi system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d ' select * from d2.t1 ' 127.0.0.1:7111/rest/sql print 31-> $system_content -if $system_content != @{"status":"succ","head":["ts","speed"],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","speed"],"column_meta":[["ts",9,8],["speed",4,4]],"data":[["2017-12-25 21:28:41.022",1]],"rows":1}@ then return -1 endi diff --git a/tests/script/general/http/restful_insert.sim b/tests/script/general/http/restful_insert.sim index f230f98723e041487232c5725b91c4755fe56fcf..b77a1dd49785bf9ad1f86e803f63e2ba0e40a8d7 100644 --- a/tests/script/general/http/restful_insert.sim +++ b/tests/script/general/http/restful_insert.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/restful_limit.sim b/tests/script/general/http/restful_limit.sim index 7d2b6e9a02fce66a3961177c635929302a92387d..48a4fdf7d3f90d8d58337992b0b8198cb43e953c 100644 --- a/tests/script/general/http/restful_limit.sim +++ b/tests/script/general/http/restful_limit.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/http/telegraf.sim b/tests/script/general/http/telegraf.sim index 4018d9661a0a1d7b445fc9eff416bd40f84d2116..9fc153b2329bac70efe2506814af4a60b89966ae 100644 --- a/tests/script/general/http/telegraf.sim +++ b/tests/script/general/http/telegraf.sim @@ -1,13 +1,13 @@ system sh/stop_dnodes.sh -sleep 3000 +sleep 2000 system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c httpEnableRecordSql -v 1 system sh/cfg.sh -n dnode1 -c telegrafUseFieldNum -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -245,7 +245,7 @@ if $system_content != @{"metrics":[{"metric":"win_cpu","stable":"win_cpu","table return -1 endi -sleep 3000 +sleep 2000 print =============== step2 - insert single data system_content curl -u root:taosdata -d '{"fields":{"Percent_DPC_Time":0,"Percent_Idle_Time":95.59830474853516,"Percent_Interrupt_Time":0,"Percent_Privileged_Time":0,"Percent_Processor_Time":0,"Percent_User_Time":0},"name":"win_cpu","tags":{"host":"windows","instance":"1","objectname":"Processor"},"timestamp":1564641722000}' 127.0.0.1:7111/telegraf/db/ @@ -285,8 +285,8 @@ system_content curl -u root:taosdata -d 'select count(*) from db.win_cpu' 127.0 print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[3]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[3]],"rows":1}@ then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/import/basic.sim b/tests/script/general/import/basic.sim index 07febb2bd52155af1cec70f84a1fe60460815750..f72d132ca39cdf0138b0c79d94857d6d646bccb5 100644 --- a/tests/script/general/import/basic.sim +++ b/tests/script/general/import/basic.sim @@ -19,14 +19,14 @@ system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 2000 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 2000 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 2000 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database ibadb diff --git a/tests/script/general/import/commit.sim b/tests/script/general/import/commit.sim index 36d201e9effaa8398f5087c95b8e03f5893a7846..3b4055d7128442ee67621ffafa9869527b7a3801 100644 --- a/tests/script/general/import/commit.sim +++ b/tests/script/general/import/commit.sim @@ -19,14 +19,14 @@ system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 2000 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 2000 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 2000 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ========= step1 @@ -72,9 +72,9 @@ endi print ========= step3 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print ========= step4 sql select * from ic2db.tb; diff --git a/tests/script/general/import/large.sim b/tests/script/general/import/large.sim index 5bf05a57fbdb3bf3c8bb123215686ccd5bdd4a8e..23fbcc75eae55623e3ed2afa8b8ebf42c77a1012 100644 --- a/tests/script/general/import/large.sim +++ b/tests/script/general/import/large.sim @@ -19,14 +19,14 @@ system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 2000 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 2000 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 2000 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db diff --git a/tests/script/general/import/replica1.sim b/tests/script/general/import/replica1.sim index d450b3fb49883df2d83b2969d951b981c30b65af..48d5455b7931bfc167112574c8193097aad74334 100644 --- a/tests/script/general/import/replica1.sim +++ b/tests/script/general/import/replica1.sim @@ -27,7 +27,7 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2 print ========= start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database ir1db days 7 @@ -93,9 +93,9 @@ endi print ================== dnode restart system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use ir1db sql select * from tb; @@ -162,9 +162,9 @@ endi print ================= step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use ir1db sql select * from tb; diff --git a/tests/script/general/insert/basic.sim b/tests/script/general/insert/basic.sim index 3f0f25a95ba68c8099c02e6daab933411089a632..88eb30a1ad819a26cdc9d61cb692fde9f1446cd7 100644 --- a/tests/script/general/insert/basic.sim +++ b/tests/script/general/insert/basic.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/insert_drop.sim b/tests/script/general/insert/insert_drop.sim index 9b68e5a6a6c7e77fc0e591e4950500369a88a650..859263762620b193873e4a355b061f12359c4f2a 100644 --- a/tests/script/general/insert/insert_drop.sim +++ b/tests/script/general/insert/insert_drop.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $tbNum = 10 @@ -43,7 +43,7 @@ print ====== tables created print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed @@ -69,7 +69,7 @@ endw print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/insert/query_block1_file.sim b/tests/script/general/insert/query_block1_file.sim index 6d6daca7b59dbd1a5fa5f1981aab6ca78691474a..636b9530f9729a98e6e290a3c360e6dfc94945ba 100644 --- a/tests/script/general/insert/query_block1_file.sim +++ b/tests/script/general/insert/query_block1_file.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_block1_memory.sim b/tests/script/general/insert/query_block1_memory.sim index bec9190f9b93044d4321b96d6604209cbc78daba..823e466ee939263ed3fdbd0bd2fdd82a597a71e0 100644 --- a/tests/script/general/insert/query_block1_memory.sim +++ b/tests/script/general/insert/query_block1_memory.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_block2_file.sim b/tests/script/general/insert/query_block2_file.sim index 34da170a9ee6cf3af09de374d8c1da54966a6382..5b7438875e8bfff0eb63ed308f7e06b2e7428da7 100644 --- a/tests/script/general/insert/query_block2_file.sim +++ b/tests/script/general/insert/query_block2_file.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_block2_memory.sim b/tests/script/general/insert/query_block2_memory.sim index 3f2c97a098c35847ae566f6d9fdba0fa85a674d4..fb41981c89cd08bc8597de5ba48d379d07858af8 100644 --- a/tests/script/general/insert/query_block2_memory.sim +++ b/tests/script/general/insert/query_block2_memory.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_file_memory.sim b/tests/script/general/insert/query_file_memory.sim index f923ebed13c2f9c592b38dc98ace644fcd8fb671..f920c215c084f288fe7ce93752ac501b43a0cb9c 100644 --- a/tests/script/general/insert/query_file_memory.sim +++ b/tests/script/general/insert/query_file_memory.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/query_multi_file.sim b/tests/script/general/insert/query_multi_file.sim index 8622fa6f9b47ff2b82305fdfff8d187d7b95c275..bbca53d309146a91dc87167756371b4ffbf2617b 100644 --- a/tests/script/general/insert/query_multi_file.sim +++ b/tests/script/general/insert/query_multi_file.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/insert/tcp.sim b/tests/script/general/insert/tcp.sim index 6f9752087d9d5d099703dd851b55305d65726784..002d84dcaea4f81adb1911e5eb9df691ee4f3605 100644 --- a/tests/script/general/insert/tcp.sim +++ b/tests/script/general/insert/tcp.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database d1; diff --git a/tests/script/general/parser/alter.sim b/tests/script/general/parser/alter.sim index eae9b88be95ed91b1e0889c8640909a6bc70f592..31d255115ee7253a73d97e1b27decd14636038d6 100644 --- a/tests/script/general/parser/alter.sim +++ b/tests/script/general/parser/alter.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect @@ -133,7 +133,7 @@ sleep 100 # return -1 #endi #sql alter table tb1 drop column c3 -#sleep 3000 +#sleep 500 #sql insert into tb1 values (now, 2, 'taos') #sleep 30000 #sql select * from strm @@ -144,7 +144,7 @@ sleep 100 # return -1 #endi #sql alter table tb1 add column c3 int -#sleep 3000 +#sleep 500 #sql insert into tb1 values (now, 3, 'taos', 3); #sleep 100 #sql select * from strm diff --git a/tests/script/general/parser/alter1.sim b/tests/script/general/parser/alter1.sim index 26ed029050641d156705cb1a22fe110ce5536ee6..a52202fc1abe044151f8af1903159bb1422e8b61 100644 --- a/tests/script/general/parser/alter1.sim +++ b/tests/script/general/parser/alter1.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect diff --git a/tests/script/general/parser/alter_stable.sim b/tests/script/general/parser/alter_stable.sim index 17fc8d984a8899c136cb8451f0fa1e9038bf2cbd..8a7f4fa924268fdace68881f86d30cbdbd131935 100644 --- a/tests/script/general/parser/alter_stable.sim +++ b/tests/script/general/parser/alter_stable.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect diff --git a/tests/script/general/parser/auto_create_tb.sim b/tests/script/general/parser/auto_create_tb.sim index 903f8f988113aded636e04512029ce67cce55e73..a902469fde690722698351f37dda7e89adff4b84 100644 --- a/tests/script/general/parser/auto_create_tb.sim +++ b/tests/script/general/parser/auto_create_tb.sim @@ -1,6 +1,6 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start @@ -208,7 +208,7 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/between_and.sim b/tests/script/general/parser/between_and.sim new file mode 100644 index 0000000000000000000000000000000000000000..cdced47cb65aea79618540b57e159b741bf9288a --- /dev/null +++ b/tests/script/general/parser/between_and.sim @@ -0,0 +1,165 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 float, f3 double, f4 bigint, f5 smallint, f6 tinyint, f7 bool, f8 binary(10), f9 nchar(10)) tags (id1 int, id2 float, id3 nchar(10), id4 double, id5 smallint, id6 bigint, id7 binary(10)) + +sql create table tb1 using st2 tags (1,1.0,"1",1.0,1,1,"1"); +sql create table tb2 using st2 tags (2,2.0,"2",2.0,2,2,"2"); +sql create table tb3 using st2 tags (3,3.0,"3",3.0,3,3,"3"); +sql create table tb4 using st2 tags (4,4.0,"4",4.0,4,4,"4"); + +sql insert into tb1 values (now-200s,1,1.0,1.0,1,1,1,true,"1","1") +sql insert into tb1 values (now-100s,2,2.0,2.0,2,2,2,true,"2","2") +sql insert into tb1 values (now,3,3.0,3.0,3,3,3,true,"3","3") +sql insert into tb1 values (now+100s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+200s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+300s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+400s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+500s,4,4.0,4.0,4,4,4,true,"4","4") + +sql select tbname,id1 from st2; + +if $rows != 4 then + return -1 +endi + + +sql select * from st2; + +if $rows != 8 then + return -1 +endi + +sql select * from st2 where ts between now-50s and now+450s + +if $rows != 5 then + return -1 +endi + +sql select tbname,id1 from st2 where id1 between 2 and 3; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3 then + return -1 +endi + +sql select tbname,id2 from st2 where id2 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2.00000 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3.00000 then + return -1 +endi + + +sql select tbname,id4 from st2 where id4 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2.000000000 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3.000000000 then + return -1 +endi + + +sql select tbname,id5 from st2 where id5 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3 then + return -1 +endi + +sql select tbname,id6 from st2 where id6 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data00 != tb2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data10 != tb3 then + return -1 +endi +if $data11 != 3 then + return -1 +endi + +sql select * from st2 where f1 between 2 and 3 and f2 between 2.0 and 3.0 and f3 between 2.0 and 3.0 and f4 between 2.0 and 3.0 and f5 between 2.0 and 3.0 and f6 between 2.0 and 3.0; + +if $rows != 2 then + return -1 +endi + +if $data01 != 2 then + return -1 +endi +if $data11 != 3 then + return -1 +endi + +sql_error select * from st2 where f7 between 2.0 and 3.0; +sql_error select * from st2 where f8 between 2.0 and 3.0; +sql_error select * from st2 where f9 between 2.0 and 3.0; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/binary_escapeCharacter.sim b/tests/script/general/parser/binary_escapeCharacter.sim index dc54add763e5cf34595c479373020c7c4baae63c..f0589d154f9fc91da50e8d83e76da1d32939bcc1 100644 --- a/tests/script/general/parser/binary_escapeCharacter.sim +++ b/tests/script/general/parser/binary_escapeCharacter.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index ae6ecb88e24a7ec10105b752b21e0714f74dcc42..8bb692e3bbe8af3ec9ed179ad29d40b4712d257b 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect @@ -93,6 +93,7 @@ $halfTbNum = $tbNum / 2 $nchar = 'nchar . $c $nchar = $nchar . ' + $ts = $ts + 1 sql insert into $tb5 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb6 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb7 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb8 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb9 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $x = $x + 1 endw @@ -105,12 +106,12 @@ run general/parser/col_arithmetic_query.sim #======================================= all in files query ======================================= print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500c +sleep 5000 run general/parser/col_arithmetic_query.sim diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 53e2c98b56e3c395e810e9edbb1d094baaa20f5c..2c56c6445fa8134ab28837940a11ff6f4127b7c7 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -41,10 +41,10 @@ if $data00 != 0.000000000 then return -1 endi -if $data01 != -nan then - print expect -nan, actual: $data01 - return -1 -endi +#if $data01 != -nan then +# print expect -nan, actual: $data01 +# return -1 +#endi if $data10 != 0.666666667 then return -1 @@ -185,9 +185,9 @@ if $data00 != 0.000000000 then return -1 endi -if $data01 != -nan then - return -1 -endi +#if $data01 != -nan then +# return -1 +#endi if $data02 != 0.000000000 then return -1 @@ -426,7 +426,7 @@ if $data02 != 9.000000020 then endi # all possible function in the arithmetic expression, add more -sql select min(c1) * max(c2) /4, sum(c1) * apercentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), count(1)/sum(c1), avg(c2)*count(c2) from $stb where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-11-25 19:30:00.000'; +sql select min(c1) * max(c2) /4, sum(c1) * apercentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), count(1)/sum(c1), avg(c2)*count(c2) from $stb where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-11-25 19:30:01.000'; if $rows != 1 then return -1 endi @@ -462,7 +462,7 @@ if $rows != 0 then endi # no result return [d.3] -sql select sum(c2) - avg(c2) from $stb where ts > '2018-11-25 19:30:00.000' +sql select sum(c2) - avg(c2) from $stb where ts > '2018-11-25 19:30:01.000' if $rows != 0 then return -1 endi @@ -520,35 +520,35 @@ if $data91 != 9 then endi # in group by column -sql select apercentile(c6, 50)-first(c6)+last(c5)*12, last(c5)*12 from ca_stb0 group by c2; -if $rows != 10 then - return -1 -endi - -if $data00 != 0.000000000 then - return -1 -endi - -if $data01 != 0.000000000 then - return -1 -endi - -if $data10 != 12.000000000 then - return -1 -endi - -if $data11 != 12.000000000 then - return -1 -endi - -if $data20 != 24.000000000 then - return -1 -endi - -if $data21 != 24.000000000 then - return -1 -endi - +#sql select apercentile(c6, 50)-first(c6)+last(c5)*12, last(c5)*12 from ca_stb0 group by c2; +#if $rows != 10 then +# return -1 +#endi +# +#if $data00 != 0.000000000 then +# return -1 +#endi +# +#if $data01 != 0.000000000 then +# return -1 +#endi +# +#if $data10 != 12.000000000 then +# return -1 +#endi +# +#if $data11 != 12.000000000 then +# return -1 +#endi +# +#if $data20 != 24.000000000 then +# return -1 +#endi +# +#if $data21 != 24.000000000 then +# return -1 +#endi +# sql_error select first(c6) - last(c6) *12 / count(*) from $stb group by c3; sql select first(c6) - last(c6) *12 / count(*) from $stb group by c5; diff --git a/tests/script/general/parser/columnValue.sim b/tests/script/general/parser/columnValue.sim index 4e6c6640042a80ffafd008538f2c46e5f43c0318..c98542fbf26a2d6098ca01c48c38d9fdca92b03f 100644 --- a/tests/script/general/parser/columnValue.sim +++ b/tests/script/general/parser/columnValue.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect diff --git a/tests/script/general/parser/columnValue_unsign.sim b/tests/script/general/parser/columnValue_unsign.sim index 895b13961ef729544a9ac14c85e4e37e6f723423..8e44ccb5facf074691f3aeeb7c60099ab6ef691f 100644 --- a/tests/script/general/parser/columnValue_unsign.sim +++ b/tests/script/general/parser/columnValue_unsign.sim @@ -83,7 +83,7 @@ if $data03 != NULL then endi sql insert into mt_unsigned_1 values(now, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); -sql insert into mt_unsigned_1 values(now, 1, 2, 3, 4, 5, 6, 7, 8, 9); +sql insert into mt_unsigned_1 values(now+1s, 1, 2, 3, 4, 5, 6, 7, 8, 9); sql_error insert into mt_unsigned_1 values(now, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); sql_error insert into mt_unsigned_1 values(now, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL); @@ -111,7 +111,8 @@ if $rows != 1 then return -1 endi -if $data00 != 6.000000000 then +if $data00 != NULL then + print expect NULL, actual:$data00 return -1 endi @@ -167,7 +168,7 @@ if $data01 != 4 then return -1 endi -// todo insert more rows and chec it +## todo insert more rows and chec it sql select first(a),count(b),last(c),sum(b),spread(d),avg(c),min(b),max(a),stddev(a) from mt_unsigned_1; if $rows != 1 then return -1 diff --git a/tests/script/general/parser/commit.sim b/tests/script/general/parser/commit.sim index 67d98de2076e7db20c95d9fd626b4a7e79c1b17f..dfe521b92bff36d50f3ec0b3ae8a82c2a9fff304 100644 --- a/tests/script/general/parser/commit.sim +++ b/tests/script/general/parser/commit.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxTablesperVnode -v 100 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -82,7 +82,7 @@ endw print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start sleep 100 print ================== server restart completed diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim index 4a8e443281dbaa0892106a8fe91bbdd6d61c3e8a..716d36e82bcc4d4505426b2c3784de03d2f77c30 100644 --- a/tests/script/general/parser/constCol.sim +++ b/tests/script/general/parser/constCol.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c dDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c mDebugFlag -v 135 diff --git a/tests/script/general/parser/create_db.sim b/tests/script/general/parser/create_db.sim index 6cf08a5ac4231ea715c1892842b0e4d122d2520c..9ca84af136ad16735c9faf4f10ba913775f53103 100644 --- a/tests/script/general/parser/create_db.sim +++ b/tests/script/general/parser/create_db.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -169,8 +169,8 @@ sql_error create database $db cache 0 sql_error create database $db ctime 29 sql_error create database $db ctime 40961 -# wal {1, 2} -sql_error create database $db wal 0 +# wal {0, 2} +#sql_error create database $db wal 0 sql_error create database $db wal -1 sql_error create database $db wal 3 diff --git a/tests/script/general/parser/create_mt.sim b/tests/script/general/parser/create_mt.sim index 9278fbfba415a89f79763b5fb8bbda67b0eb89c5..ae1629dce9861d7540bda4b6d4014e32ce2ce52d 100644 --- a/tests/script/general/parser/create_mt.sim +++ b/tests/script/general/parser/create_mt.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -126,8 +126,7 @@ $tb_ = table $tbs = tables $db_ = database $dbs = databases -$mt_ = metric -$mts = metrics +$ses = session $int = int $bint = bigint $binary = binary @@ -145,8 +144,7 @@ sql_error create table $mt (ts timestamp, col1 int) tags ( $tb_ int) sql_error create table $mt (ts timestamp, col1 int) tags ( $tbs int) sql_error create table $mt (ts timestamp, col1 int) tags ( $db_ int) sql_error create table $mt (ts timestamp, col1 int) tags ( $dbs int) -sql_error create table $mt (ts timestamp, col1 int) tags ( $mt_ int) -sql_error create table $mt (ts timestamp, col1 int) tags ( $mts int) +sql_error create table $mt (ts timestamp, col1 int) tags ( $ses int) sql_error create table $mt (ts timestamp, col1 int) tags ( $int int) sql_error create table $mt (ts timestamp, col1 int) tags ( $bint int) sql_error create table $mt (ts timestamp, col1 int) tags ( $binary int) diff --git a/tests/script/general/parser/create_tb.sim b/tests/script/general/parser/create_tb.sim index 48b7a0fb1994f192cf984b37665fdc6281e9937f..eb6e4f71c3f752d0e54ba275f2e8f04dc5b08e65 100644 --- a/tests/script/general/parser/create_tb.sim +++ b/tests/script/general/parser/create_tb.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -90,8 +90,7 @@ $tb_ = table $tbs = tables $db_ = database $dbs = databases -$mt_ = metric -$mts = metrics +$ses = session $int = int $bint = bigint $binary = binary @@ -105,9 +104,8 @@ $nchar = nchar sql_error create table $tb (ts timestamp, $tb_ int) sql_error create table $tb (ts timestamp, $tbs int) sql_error create table $tb (ts timestamp, $db_ int) -sql_error create table $tb (ts timestamp, $dbs int) -sql_error create table $tb (ts timestamp, $mt_ int) -sql_error create table $tb (ts timestamp, $mts int) +sql_error create table $tb (ts timestamp, $dbs int) +sql_error create table $tb (ts timestamp, $ses int) sql_error create table $tb (ts timestamp, $int int) sql_error create table $tb (ts timestamp, $bint int) sql_error create table $tb (ts timestamp, $binary int) diff --git a/tests/script/general/parser/create_tb_with_tag_name.sim b/tests/script/general/parser/create_tb_with_tag_name.sim new file mode 100644 index 0000000000000000000000000000000000000000..130f4097f6547e50127afd86d8d73902a5653ae2 --- /dev/null +++ b/tests/script/general/parser/create_tb_with_tag_name.sim @@ -0,0 +1,162 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db +sql use $db + +sql create stable st2 (ts timestamp, f1 int) tags (id int, t1 int, t2 nchar(4), t3 double) + + +sql insert into tb1 using st2 (id, t1) tags(1,2) values (now, 1) + +sql select id,t1,t2,t3 from tb1 + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != NULL then + return -1 +endi +if $data03 != NULL then + return -1 +endi + +sql create table tb2 using st2 (t2,t3) tags ("12",22.0) + +sql select id,t1,t2,t3 from tb2; + +if $rows != 1 then + return -1 +endi + +if $data00 != NULL then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 12 then + return -1 +endi +if $data03 != 22.000000000 then + return -1 +endi + + +sql create table tb3 using st2 tags (1,2,"3",33.0); + +sql select id,t1,t2,t3 from tb3; + + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 3 then + return -1 +endi +if $data03 != 33.000000000 then + return -1 +endi + +sql insert into tb4 using st2 tags(1,2,"33",44.0) values (now, 1); + +sql select id,t1,t2,t3 from tb4; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 33 then + return -1 +endi +if $data03 != 44.000000000 then + return -1 +endi + +sql_error create table tb5 using st2() tags (3,3,"3",33.0); + +sql_error create table tb6 using st2 (id,t1) tags (3,3,"3",33.0); + +sql_error create table tb7 using st2 (id,t1) tags (3); + +sql_error create table tb8 using st2 (ide) tags (3); + +sql_error create table tb9 using st2 (id); + +sql_error create table tb10 using st2 (id t1) tags (1,1); + +sql_error create table tb10 using st2 (id,,t1) tags (1,1,1); + +sql_error create table tb11 using st2 (id,t1,) tags (1,1,1); + +sql create table tb12 using st2 (t1,id) tags (2,1); + +sql select id,t1,t2,t3 from tb12; +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != NULL then + return -1 +endi +if $data03 != NULL then + return -1 +endi + +sql create table tb13 using st2 ("t1",'id') tags (2,1); + +sql select id,t1,t2,t3 from tb13; + +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != NULL then + return -1 +endi +if $data03 != NULL then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/dbtbnameValidate.sim b/tests/script/general/parser/dbtbnameValidate.sim index 072fd740d4d2960891c774a4e24a999c477ebc96..f2e6de81f1cd6bedf3b455bb35b68f669cd889e1 100644 --- a/tests/script/general/parser/dbtbnameValidate.sim +++ b/tests/script/general/parser/dbtbnameValidate.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect @@ -12,25 +12,25 @@ sql create database 'abc123' sql create database '_ab1234' sql create database 'ABC123' sql create database '_ABC123' -sql create database 'aABb123 ' -sql create database ' xyz ' -sql create database ' XYZ ' +sql_error create database 'aABb123 ' +sql_error create database ' xyz ' +sql_error create database ' XYZ ' sql use 'abc123' sql use '_ab1234' sql use 'ABC123' sql use '_ABC123' -sql use 'aABb123' -sql use ' xyz ' -sql use ' XYZ ' +sql_error use 'aABb123' +sql_error use ' xyz ' +sql_error use ' XYZ ' sql drop database 'abc123' sql drop database '_ab1234' -sql drop database 'ABC123' +sql_error drop database 'ABC123' sql drop database '_ABC123' -sql drop database 'aABb123' -sql drop database ' xyz ' -sql drop database ' XYZ ' +sql_error drop database 'aABb123' +sql_error drop database ' xyz ' +sql_error drop database ' XYZ ' sql use abc @@ -67,9 +67,9 @@ sql describe mt sql describe sub_001 sql describe sub_dy_tbl -sql_error describe Dd -sql_error describe FF -sql_error describe gG +sql describe Dd +sql describe FF +sql describe gG sql drop table abc.cc sql drop table 'abc.Dd' @@ -119,4 +119,4 @@ if $rows != 4 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim index aac79e1a3ca5a52a89d4f29ba43efc7be228c0f9..d109dd50f7c0ba0684295dbf093ba1b280e04fce 100644 --- a/tests/script/general/parser/fill.sim +++ b/tests/script/general/parser/fill.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect @@ -313,6 +313,7 @@ if $rows != 9 then return -1 endi if $data01 != 0 then + print expect 0, actual:$data01 return -1 endi if $data11 != 6 then @@ -979,10 +980,6 @@ if $data00 != @20-01-01 01:01:00.000@ then return -1 endi -if $data00 != @20-01-01 01:01:00.000@ then - return -1 -endi -if $data1 if $data01 != 2.000000000 then return -1 endi diff --git a/tests/script/general/parser/fill_stb.sim b/tests/script/general/parser/fill_stb.sim index a9547b8a9408a52aa3cda64301326b81c2e4655f..ba8ddbdf6ac6e9035398b3ac7c26861c02771e99 100644 --- a/tests/script/general/parser/fill_stb.sim +++ b/tests/script/general/parser/fill_stb.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect diff --git a/tests/script/general/parser/fill_us.sim b/tests/script/general/parser/fill_us.sim index a429df059bad3cd36301ecd4685db89d74090ba0..8cd2c333475a0d0140eb5c0c8ee0fa4186fccc97 100644 --- a/tests/script/general/parser/fill_us.sim +++ b/tests/script/general/parser/fill_us.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim index a16b5b1e077f3fe3ffced5a6809bf7ca10423606..f47f67ddc668c94f423cb3a42d97be3d52c8dcda 100644 --- a/tests/script/general/parser/first_last.sim +++ b/tests/script/general/parser/first_last.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxTablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -77,7 +77,7 @@ run general/parser/first_last_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 7d702e989e6f816832adbd92507efbef68fdb088..9165b7e98e0510c3fead45acbc1baeceb0b653a3 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect @@ -382,4 +382,417 @@ sql drop table cars; sql create table cars(ts timestamp, c int) tags(id int); sql create table car1 using cars tags(1); sql create table car2 using cars tags(2); -sql insert into car1 (ts, c) values (now,1) car2(ts, c) values(now, 2); \ No newline at end of file +sql insert into car1 (ts, c) values (now,1) car2(ts, c) values(now, 2); + +print ========================> TD-2700 +sql create table tx(ts timestamp, k int); +sql insert into tx values(1500000001000, 0); +sql select sum(k) from tx interval(1d) sliding(1h); +if $rows != 24 then + print expect 24, actual:$rows + return -1 +endi + +print ========================> TD-2740 +sql drop table if exists m1; +sql create table m1(ts timestamp, k int) tags(a int); +sql create table tm10 using m1 tags(0); +sql create table tm11 using m1 tags(1); +sql create table tm12 using m1 tags(2); +sql create table tm13 using m1 tags(3); +sql insert into tm10 values('2020-1-1 1:1:1', 0); +sql insert into tm11 values('2020-1-5 1:1:1', 0); +sql insert into tm12 values('2020-1-7 1:1:1', 0); +sql insert into tm13 values('2020-1-1 1:1:1', 0); +sql select count(*) from m1 where ts='2020-1-1 1:1:1' interval(1h) group by tbname; +if $rows != 2 then + return -1 +endi + +sql drop table m1; +sql drop table if exists tm1; +sql drop table if exists tm2; +sql create table m1(ts timestamp, k double, b double, c int, d smallint, e int unsigned) tags(a int); +sql create table tm1 using m1 tags(1); +sql create table tm2 using m1 tags(2); +sql insert into tm1 values('2021-01-27 22:22:39.294', 1, 10, NULL, 110, 123) ('2021-01-27 22:22:40.294', 2, 20, NULL, 120, 124) ('2021-01-27 22:22:41.294', 3, 30, NULL, 130, 125)('2021-01-27 22:22:43.294', 4, 40, NULL, 140, 126)('2021-01-27 22:22:44.294', 5, 50, NULL, 150, 127); +sql insert into tm2 values('2021-01-27 22:22:40.688', 5, 101, NULL, 210, 321) ('2021-01-27 22:22:41.688', 5, 102, NULL, 220, 322) ('2021-01-27 22:22:42.688', 5, 103, NULL, 230, 323)('2021-01-27 22:22:43.688', 5, 104, NULL, 240, 324)('2021-01-27 22:22:44.688', 5, 105, NULL, 250, 325)('2021-01-27 22:22:45.688', 5, 106, NULL, 260, 326); +sql select stddev(k) from m1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.378704626 then + return -1 +endi + +sql select stddev(c) from m1 +if $rows != 0 then + return -1 +endi + +sql select stddev(k), stddev(c) from m1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.378704626 then + return -1 +endi + +if $data01 != NULL then + return -1; +endi + +sql select stddev(b),stddev(b),stddev(k) from m1; +if $rows != 1 then + return -1 +endi + +if $data00 != 37.840465463 then + return -1 +endi + +if $data01 != 37.840465463 then + return -1 +endi + +if $data02 != 1.378704626 then + return -1 +endi + +sql select stddev(k), stddev(b) from m1 group by a +if $rows != 2 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 14.142135624 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data10 != 0.000000000 then + return -1 +endi + +if $data11 != 1.707825128 then + return -1 +endi + +if $data12 != 2 then + return -1 +endi + +sql select stddev(k), stddev(b) from m1 where a= 1 group by a +if $rows != 1 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 14.142135624 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select stddev(k), stddev(b) from m1 group by tbname +if $rows != 2 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 14.142135624 then + return -1 +endi + +if $data02 != @tm1@ then + return -1 +endi + +if $data10 != 0.000000000 then + return -1 +endi + +if $data11 != 1.707825128 then + return -1 +endi + +if $data12 != @tm2@ then + return -1 +endi + +sql select stddev(k), stddev(b) from m1 group by tbname,a +if $rows != 2 then + return -1 +endi + +sql select stddev(k), stddev(b), stddev(c) from m1 group by tbname,a +if $rows != 2 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 14.142135624 then + return -1 +endi + +if $data02 != NULL then + return -1 +endi + +if $data03 != @tm1@ then + return -1 +endi + +if $data04 != 1 then + return -1 +endi + +if $data10 != 0.000000000 then + return -1 +endi + +if $data11 != 1.707825128 then + return -1 +endi + +if $data12 != NULL then + return -1 +endi + +if $data13 != @tm2@ then + return -1 +endi + +if $data14 != 2 then + return -1 +endi + +sql select stddev(k), stddev(b), stddev(c) from m1 interval(10s) group by tbname,a +if $rows != 3 then + return -1 +endi + +if $data01 != 0.000000000 then + return -1 +endi + +if $data02 != 0.000000000 then + return -1 +endi + +if $data03 != NULL then + return -1 +endi + +if $data04 != @tm1@ then + return -1 +endi + +if $data05 != 1 then + return -1 +endi + +if $data11 != 1.118033989 then + return -1 +endi + +if $data12 != 11.180339887 then + return -1 +endi + +if $data13 != NULL then + return -1 +endi + +if $data14 != @tm1@ then + return -1 +endi + +if $data22 != 1.707825128 then + return -1 +endi + +if $data23 != NULL then + return -1 +endi + +if $data24 != @tm2@ then + return -1 +endi + +if $data25 != 2 then + return -1 +endi + +sql select count(*), first(b), stddev(b), stddev(c) from m1 interval(10s) group by a +if $rows != 3 then + return -1 +endi + +if $data00 != @21-01-27 22:22:30.000@ then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 10.000000000 then + return -1 +endi + +if $data03 != 0.000000000 then + return -1 +endi + +if $data04 != NULL then + return -1 +endi + +if $data05 != 1 then + return -1 +endi + +if $data12 != 20.000000000 then + return -1 +endi + +if $data13 != 11.180339887 then + return -1 +endi + +if $data14 != NULL then + return -1 +endi + +if $data23 != 1.707825128 then + return -1 +endi + +sql select count(*), first(b), stddev(b), stddev(c) from m1 interval(10s) group by tbname,a +if $rows != 3 then + return -1 +endi + +if $data23 != 1.707825128 then + return -1 +endi + +if $data25 != @tm2@ then + return -1 +endi + +sql select count(*), stddev(b), stddev(b)+20, stddev(c) from m1 interval(10s) group by tbname,a +if $rows != 3 then + return -1 +endi + +if $data02 != 0.000000000 then + return -1 +endi + +if $data03 != 20.000000000 then + return -1 +endi + +if $data13 != 31.180339887 then + return -1 +endi + +if $data14 != NULL then + return -1 +endi + +sql select count(*), first(b), stddev(b)+first(b), stddev(c) from m1 interval(10s) group by tbname,a +if $rows != 3 then + return -1 +endi + +if $data02 != 10.000000000 then + return -1 +endi + +if $data03 != 10.000000000 then + return -1 +endi + +if $data12 != 20.000000000 then + return -1 +endi + +if $data13 != 31.180339887 then + return -1 +endi + +if $data22 != 101.000000000 then + return -1 +endi + +if $data23 != 102.707825128 then + return -1 +endi + +sql select stddev(e),stddev(k) from m1 where a=1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1.414213562 then + return -1 +endi + +if $data01 != 1.414213562 then + return -1 +endi + +sql create stable st1 (ts timestamp, f1 int, f2 int) tags (id int); +sql create table tb1 using st1 tags(1); + +sql insert into tb1 values (now, 1, 1); + +sql select stddev(f1) from st1 group by f1; + +if $rows != 1 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +sql select count(tbname) from st1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select count(id) from st1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 19b14e327c0cd180957748b656b8ad15c0ad35cd..dd7331054c2ee6edd3ba07327057404dae7b3ed7 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -610,6 +610,11 @@ print =================>TD-2665 sql_error create table txx as select avg(c) as t from st; sql_error create table txx1 as select avg(c) as t from t1; +sql select stddev(c),stddev(c) from st group by c; +if $rows != 4 then + return -1 +endi + print =================>TD-2236 sql select first(ts),last(ts) from t1 group by c; if $rows != 4 then diff --git a/tests/script/general/parser/import.sim b/tests/script/general/parser/import.sim index 83751dc6166275d7d61ad9d63b41b0817ecd8a77..4468ab87a923fc65eeb22eff97dc7d56bfdc7dc9 100644 --- a/tests/script/general/parser/import.sim +++ b/tests/script/general/parser/import.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect @@ -25,15 +25,15 @@ sql use $db sql create table tb (ts timestamp, c1 int, c2 timestamp) sql insert into tb values ('2019-05-05 11:30:00.000', 1, now) sql insert into tb values ('2019-05-05 12:00:00.000', 1, now) -sleep 2000 +sleep 500 sql import into tb values ('2019-05-05 11:00:00.000', -1, now) -sleep 2000 +sleep 500 sql import into tb values ('2019-05-05 11:59:00.000', -1, now) -sleep 2000 +sleep 500 sql import into tb values ('2019-05-04 08:00:00.000', -1, now) -sleep 2000 +sleep 500 sql import into tb values ('2019-05-04 07:59:00.000', -1, now) -sleep 2000 +sleep 500 sql select * from tb if $rows != 6 then @@ -60,7 +60,7 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 2000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/import_commit1.sim b/tests/script/general/parser/import_commit1.sim index eb49be947c71727a1367026420728cbd94fca7d4..f330fe4cd94b1cba82ea985b9b4841245753e099 100644 --- a/tests/script/general/parser/import_commit1.sim +++ b/tests/script/general/parser/import_commit1.sim @@ -40,7 +40,7 @@ while $x < $rowNum endw print ====== tables created -sleep 3000 +sleep 500 $ts = $ts0 + $delta $ts = $ts + 1 diff --git a/tests/script/general/parser/import_commit2.sim b/tests/script/general/parser/import_commit2.sim index 7222a5412be6b62d90f88878179e8e49f39fa661..47b30acb49111da69c1c0b8b1927fd7f13f0c31c 100644 --- a/tests/script/general/parser/import_commit2.sim +++ b/tests/script/general/parser/import_commit2.sim @@ -39,7 +39,7 @@ while $x < $rowNum endw print ====== tables created -sleep 3000 +sleep 500 $ts = $ts0 + $delta $ts = $ts + 1 diff --git a/tests/script/general/parser/import_commit3.sim b/tests/script/general/parser/import_commit3.sim index ea9980930a2ea6c626c04b8bfaa5b3d5eb863a0a..1e041375de3da7fc2a5c17ae40caa1251b5d9d1b 100644 --- a/tests/script/general/parser/import_commit3.sim +++ b/tests/script/general/parser/import_commit3.sim @@ -39,7 +39,7 @@ while $x < $rowNum endw print ====== tables created -sleep 3000 +sleep 500 $ts = $ts + 1 sql insert into $tb values ( $ts , -1, -1, -1, -1, -1) @@ -47,7 +47,7 @@ $ts = $ts0 + $delta $ts = $ts + 1 sql import into $tb values ( $ts , -2, -2, -2, -2, -2) -sleep 3000 +sleep 500 sql show databases diff --git a/tests/script/general/parser/import_file.sim b/tests/script/general/parser/import_file.sim index 6b4dd07c7981ca141765b41cca3d005874a6583f..217679047aa823119269d6a082270d75b9e2975f 100644 --- a/tests/script/general/parser/import_file.sim +++ b/tests/script/general/parser/import_file.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 +sleep 500 sql connect -sleep 2000 +sleep 500 sql drop database if exists indb diff --git a/tests/script/general/parser/insert_multiTbl.sim b/tests/script/general/parser/insert_multiTbl.sim index 39223d84e394454b028b40a8dfa3f5457e0e9428..b17323280eb0297fc648fc6d06b38856b7a2299e 100644 --- a/tests/script/general/parser/insert_multiTbl.sim +++ b/tests/script/general/parser/insert_multiTbl.sim @@ -2,9 +2,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 2000 +sleep 500 sql connect sleep 100 print ======================== dnode1 start diff --git a/tests/script/general/parser/insert_tb.sim b/tests/script/general/parser/insert_tb.sim index f212325f26315f815a1a9f69ea84844b57579081..1e431aef3dc8355b4766abb81db6a94ee34052bf 100644 --- a/tests/script/general/parser/insert_tb.sim +++ b/tests/script/general/parser/insert_tb.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 diff --git a/tests/script/general/parser/interp.sim b/tests/script/general/parser/interp.sim index 4078fc1ead096a34e3bbc4c6b276a6a16f0a21a7..3fb91e36c66985d90776b33b89607fa9a272d500 100644 --- a/tests/script/general/parser/interp.sim +++ b/tests/script/general/parser/interp.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect @@ -59,7 +59,7 @@ run general/parser/interp_test.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/interp_test.sim b/tests/script/general/parser/interp_test.sim index 819e5741d3966c81b52c0f2626d7004ad911981c..81a77995fb828db00b65d085e0839e3b652385e0 100644 --- a/tests/script/general/parser/interp_test.sim +++ b/tests/script/general/parser/interp_test.sim @@ -579,7 +579,7 @@ $tb = $tbPrefix . 0 ## interp(*) from stb + group by + fill(none) $t = $ts0 + 1000 - sql select interp(*) from $stb where ts = $t fill(NULL) group by tbname +sql select interp(*) from $stb where ts = $t fill(NULL) group by tbname if $rows != $tbNum then return -1 endi @@ -638,209 +638,293 @@ if $data24 != NULL then return -1 endi - ## interp(*) from stb + group by + fill(prev) - $t = $ts0 + 1000 - sql select interp(*) from $stb where ts = $t fill(prev) group by tbname - print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 - print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 - if $rows != $tbNum then - return -1 - endi - if $data00 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data01 != 0 then - return -1 - endi - if $data02 != 0 then - return -1 - endi - if $data03 != 0.00000 then - return -1 - endi - if $data04 != 0.000000000 then - return -1 - endi - if $data05 != 0 then - return -1 - endi - if $data06 != 0 then - return -1 - endi - if $data07 != 1 then - return -1 - endi - if $data08 != binary0 then - return -1 - endi - if $data09 != nchar0 then - return -1 - endi - if $data20 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data21 != 0 then - return -1 - endi - if $data22 != NULL then - return -1 - endi - if $data23 != 0.00000 then - return -1 - endi - if $data24 != NULL then - return -1 - endi - if $data25 != 0 then - return -1 - endi - if $data26 != 0 then - return -1 - endi - if $data27 != 1 then - return -1 - endi - if $data28 != binary0 then - return -1 - endi - if $data29 != nchar0 then - return -1 - endi +## interp(*) from stb + group by + fill(prev) +$t = $ts0 + 1000 +sql select interp(*) from $stb where ts = $t fill(prev) group by tbname +print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 +print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 +if $rows != $tbNum then + return -1 +endi +if $data00 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data01 != 0 then + return -1 +endi +if $data02 != 0 then + return -1 +endi +if $data03 != 0.00000 then + return -1 +endi +if $data04 != 0.000000000 then + return -1 +endi +if $data05 != 0 then + return -1 +endi +if $data06 != 0 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != binary0 then + return -1 +endi +if $data09 != nchar0 then + return -1 +endi +if $data20 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data21 != 0 then + return -1 +endi +if $data22 != NULL then + return -1 +endi +if $data23 != 0.00000 then + return -1 +endi +if $data24 != NULL then + return -1 +endi +if $data25 != 0 then + return -1 +endi +if $data26 != 0 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != binary0 then + return -1 +endi +if $data29 != nchar0 then + return -1 +endi - ## interp(*) from stb + group by + fill(linear) - $t = $ts0 + 1000 - sql select interp(*) from $stb where ts = $t fill(linear) group by tbname - print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 - print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 - if $rows != $tbNum then - return -1 - endi - if $data00 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data01 != 0 then - return -1 - endi - if $data02 != 0 then - return -1 - endi - if $data03 != 0.00167 then - return -1 - endi - if $data04 != 0.001666667 then - return -1 - endi - if $data05 != 0 then - return -1 - endi - if $data06 != 0 then - return -1 - endi - if $data07 != NULL then - return -1 - endi - if $data08 != NULL then - return -1 - endi - if $data09 != NULL then - return -1 - endi - if $data20 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data21 != 0 then - return -1 - endi - if $data22 != NULL then - return -1 - endi - if $data23 != 0.00167 then - return -1 - endi - if $data24 != NULL then - return -1 - endi - if $data25 != 0 then - return -1 - endi - if $data26 != 0 then - return -1 - endi - if $data27 != NULL then - return -1 - endi - if $data28 != NULL then - return -1 - endi - if $data29 != NULL then - return -1 - endi +## interp(*) from stb + group by + fill(linear) +$t = $ts0 + 1000 +sql select interp(*) from $stb where ts = $t fill(linear) group by tbname +print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 +print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 +if $rows != $tbNum then + return -1 +endi +if $data00 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data01 != 0 then + return -1 +endi +if $data02 != 0 then + return -1 +endi +if $data03 != 0.00167 then + return -1 +endi +if $data04 != 0.001666667 then + return -1 +endi +if $data05 != 0 then + return -1 +endi +if $data06 != 0 then + return -1 +endi +if $data07 != NULL then + return -1 +endi +if $data08 != NULL then + return -1 +endi +if $data09 != NULL then + return -1 +endi +if $data20 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data21 != 0 then + return -1 +endi +if $data22 != NULL then + return -1 +endi +if $data23 != 0.00167 then + return -1 +endi +if $data24 != NULL then + return -1 +endi +if $data25 != 0 then + return -1 +endi +if $data26 != 0 then + return -1 +endi +if $data27 != NULL then + return -1 +endi +if $data28 != NULL then + return -1 +endi +if $data29 != NULL then + return -1 +endi - ## interp(*) from stb + group by + fill(value) - $t = $ts0 + 1000 - sql select interp(*) from $stb where ts = $t fill(value, -1, -2) group by tbname - print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 - print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 - if $rows != $tbNum then - return -1 - endi - if $data00 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data01 != -2 then - return -1 - endi - if $data02 != -2 then - return -1 - endi - if $data03 != -2.00000 then - return -1 - endi - if $data04 != -2.000000000 then - return -1 - endi - if $data05 != -2 then - return -1 - endi - if $data06 != -2 then - return -1 - endi - if $data07 != 1 then - return -1 - endi - if $data08 != NULL then - return -1 - endi - if $data09 != NULL then - return -1 - endi - if $data20 != @18-09-17 09:00:01.000@ then - return -1 - endi - if $data21 != -2 then - return -1 - endi - if $data22 != -2 then - return -1 - endi - if $data23 != -2.00000 then - return -1 - endi - if $data24 != -2.000000000 then - return -1 - endi - if $data25 != -2 then - return -1 - endi - if $data26 != -2 then - return -1 - endi - if $data27 != 1 then - return -1 - endi - if $data28 != NULL then - return -1 - endi - if $data29 != NULL then - return -1 - endi + ## interp(*) from stb + group by + fill(value) +$t = $ts0 + 1000 +sql select interp(*) from $stb where ts = $t fill(value, -1, -2) group by tbname +print ====== 0:$data00, 1:$data01, 2:$data02, 3:$data03, 4:$data04, 5:$data05, 6:$data06, 7:$data07, 8:$data08, 9:$data09 +print ====== 0:$data20, 1:$data21, 2:$data22, 3:$data23, 4:$data24, 5:$data25, 6:$data26, 7:$data27, 8:$data28, 9:$data29 +if $rows != $tbNum then + return -1 +endi +if $data00 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data01 != -2 then + return -1 +endi +if $data02 != -2 then + return -1 +endi +if $data03 != -2.00000 then + return -1 +endi +if $data04 != -2.000000000 then + return -1 +endi +if $data05 != -2 then + return -1 +endi +if $data06 != -2 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != NULL then + return -1 +endi +if $data09 != NULL then + return -1 +endi +if $data20 != @18-09-17 09:00:01.000@ then + return -1 +endi +if $data21 != -2 then + return -1 +endi +if $data22 != -2 then + return -1 +endi +if $data23 != -2.00000 then + return -1 +endi +if $data24 != -2.000000000 then + return -1 +endi +if $data25 != -2 then + return -1 +endi +if $data26 != -2 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != NULL then + return -1 +endi +if $data29 != NULL then + return -1 +endi + +sql_error select interp(ts,c1) from intp_tb0 where ts>'2018-11-25 19:19:00' and ts<'2018-11-25 19:19:12'; +sql select interp(ts,c1) from intp_tb0 where ts>'2018-11-25 19:19:00' and ts<'2018-11-25 19:19:12' interval(1s) fill(linear); +if $rows != 0 then + return -1 +endi + +sql select interp(c1) from intp_tb0 where ts>'2018-11-25 18:09:00' and ts<'2018-11-25 19:20:12' interval(18m); +if $rows != 1 then + return -1 +endi + +if $data00 != @18-11-25 18:30:00.000@ then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select interp(c1,c3,c4,ts) from intp_tb0 where ts>'2018-11-25 18:09:00' and ts<'2018-11-25 19:20:12' interval(18m) fill(linear) +if $rows != 5 then + return -1 +endi + +if $data00 != @18-11-25 17:54:00.000@ then + return -1 +endi + +if $data01 != 0 then + return -1 +endi + +if $data02 != 0.00000 then + return -1 +endi + +if $data03 != 0.000000000 then + return -1 +endi + +if $data04 != @18-11-25 17:54:00.000@ then + return -1 +endi + +if $data10 != @18-11-25 18:12:00.000@ then + return -1 +endi + +if $data11 != 1 then + return -1 +endi + +if $data12 != 1.20000 then + return -1 +endi + +if $data13 != 1.200000000 then + return -1 +endi + +if $data14 != @18-11-25 18:12:00.000@ then + return -1 +endi + +if $data40 != @18-11-25 19:06:00.000@ then + return -1 +endi + +if $data41 != 6 then + return -1 +endi + +if $data42 != 6.60000 then + return -1 +endi + +if $data43 != 6.600000000 then + return -1 +endi + +if $data44 != @18-11-25 19:06:00.000@ then + return -1 +endi \ No newline at end of file diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index d18a3d76760bb61f7ddd5d37f37b073ca40ffba6..2c14a86c3a6532088d0dce584199c0d5ffbed6fd 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c debugFlag -v 135 system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 @@ -347,6 +347,7 @@ $val = $rowNum + $rowNum print $val print $rows if $rows != $val then + print expect $val , actual:$rows return -1 endi @@ -415,6 +416,7 @@ sql select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_m $val = 100 if $rows != $val then + print $rows return -1 endi @@ -446,7 +448,7 @@ endi sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb0.c7 = true #====================group by========================================= - +print =================>"group by not supported" @@ -514,4 +516,4 @@ sql drop table tm2; sql select count(*) from m1, m2 where m1.ts=m2.ts and m1.b=m2.a; sql drop database ux1; -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/join_manyblocks.sim b/tests/script/general/parser/join_manyblocks.sim new file mode 100644 index 0000000000000000000000000000000000000000..fddd59c0a148975fd1bb6d63dbfb01ab46baa376 --- /dev/null +++ b/tests/script/general/parser/join_manyblocks.sim @@ -0,0 +1,93 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 + +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 100 + +$dbPrefix = join_m_db +$tbPrefix = join_tb +$mtPrefix = join_mt +$tbNum = 3 +$rowNum = 20000 +$totalNum = $tbNum * $rowNum + +print =============== join_manyBlocks.sim +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +$tstart = 100000 + +sql drop database if exists $db -x step1 +step1: +sql create database if not exists $db keep 36500 + +sql use $db +sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12)) + +$mt1 = $mtPrefix . 1 . $i +sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12), t3 int) + +$i = 0 +$tbPrefix1 = join_1_tb + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + $tg2 = ' . abc + $tg2 = $tg2 . ' + sql create table $tb using $mt tags( $i , $tg2 ) + + $tb1 = $tbPrefix1 . $i + $c = $i + $t3 = $i + 1 + + $binary = ' . abc + $binary = $binary . $i + $binary = $binary . ' + + print $binary + sql create table $tb1 using $mt1 tags( $i , $binary , $t3 ) + + $x = 0 + while $x < $rowNum + $ms = $x . m + $c = $x / 100 + $c = $c * 100 + $c = $x - $c + + $binary = ' . binary + $binary = $binary . $c + $binary = $binary . ' + + $nchar = ' . nchar + $nchar = $nchar . $c + $nchar = $nchar . ' + + sql insert into $tb values ($tstart , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) $tb1 values ($tstart , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 100000 +endw + +sleep 100 + +print ===============join_manyblocks.sim +print ==============> td-3313 +sql select join_mt0.ts,join_mt0.ts,join_mt0.t1 from join_mt0, join_mt1 where join_mt0.ts=join_mt1.ts and join_mt0.t1=join_mt1.t1; + +print $row +if $row != 60000 then + print expect 60000, actual: $row + return -1 +endi + +print ======= second tags join diff --git a/tests/script/general/parser/join_multitables.sim b/tests/script/general/parser/join_multitables.sim new file mode 100644 index 0000000000000000000000000000000000000000..acb8be10e7cf0f4a3f70828b1054d9552ca864c4 --- /dev/null +++ b/tests/script/general/parser/join_multitables.sim @@ -0,0 +1,2326 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db +sql use $db + +sql create stable st0 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st1 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st2 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st3 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st4 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st5 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st6 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st7 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st8 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable st9 (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable sta (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); +sql create stable stb (ts timestamp, f1 int, f2 double, f3 binary(10)) tags(id1 int, id2 smallint, id3 double, id4 bool, id5 binary(5)); + +sql create table tb0_1 using st0 tags(0,1,2.0,true,'3'); +sql create table tb0_2 using st0 tags(1,2,3.0,false,'4'); +sql create table tb0_3 using st0 tags(2,3,4.0,true,'5'); +sql create table tb0_4 using st0 tags(3,4,5.0,false,'6'); +sql create table tb0_5 using st0 tags(4,5,6.0,true,'7'); + +sql create table tb1_1 using st1 tags(0,1,2.0,true,'3'); +sql create table tb1_2 using st1 tags(1,2,3.0,false,'4'); +sql create table tb1_3 using st1 tags(2,3,4.0,true,'5'); +sql create table tb1_4 using st1 tags(3,4,5.0,false,'6'); +sql create table tb1_5 using st1 tags(4,5,6.0,true,'7'); + +sql create table tb2_1 using st2 tags(0,1,2.0,true,'3'); +sql create table tb2_2 using st2 tags(1,2,3.0,false,'4'); +sql create table tb2_3 using st2 tags(2,3,4.0,true,'5'); +sql create table tb2_4 using st2 tags(3,4,5.0,false,'6'); +sql create table tb2_5 using st2 tags(4,5,6.0,true,'7'); + +sql create table tb3_1 using st3 tags(0,1,2.0,true,'3'); +sql create table tb3_2 using st3 tags(1,2,3.0,false,'4'); +sql create table tb3_3 using st3 tags(2,3,4.0,true,'5'); +sql create table tb3_4 using st3 tags(3,4,5.0,false,'6'); +sql create table tb3_5 using st3 tags(4,5,6.0,true,'7'); + +sql create table tb4_1 using st4 tags(0,1,2.0,true,'3'); +sql create table tb4_2 using st4 tags(1,2,3.0,false,'4'); +sql create table tb4_3 using st4 tags(2,3,4.0,true,'5'); +sql create table tb4_4 using st4 tags(3,4,5.0,false,'6'); +sql create table tb4_5 using st4 tags(4,5,6.0,true,'7'); + +sql create table tb5_1 using st5 tags(0,1,2.0,true,'3'); +sql create table tb5_2 using st5 tags(1,2,3.0,false,'4'); +sql create table tb5_3 using st5 tags(2,3,4.0,true,'5'); +sql create table tb5_4 using st5 tags(3,4,5.0,false,'6'); +sql create table tb5_5 using st5 tags(4,5,6.0,true,'7'); + +sql create table tb6_1 using st6 tags(0,1,2.0,true,'3'); +sql create table tb6_2 using st6 tags(1,2,3.0,false,'4'); +sql create table tb6_3 using st6 tags(2,3,4.0,true,'5'); +sql create table tb6_4 using st6 tags(3,4,5.0,false,'6'); +sql create table tb6_5 using st6 tags(4,5,6.0,true,'7'); + +sql create table tb7_1 using st7 tags(0,1,2.0,true,'3'); +sql create table tb7_2 using st7 tags(1,2,3.0,false,'4'); +sql create table tb7_3 using st7 tags(2,3,4.0,true,'5'); +sql create table tb7_4 using st7 tags(3,4,5.0,false,'6'); +sql create table tb7_5 using st7 tags(4,5,6.0,true,'7'); + +sql create table tb8_1 using st8 tags(0,1,2.0,true,'3'); +sql create table tb8_2 using st8 tags(1,2,3.0,false,'4'); +sql create table tb8_3 using st8 tags(2,3,4.0,true,'5'); +sql create table tb8_4 using st8 tags(3,4,5.0,false,'6'); +sql create table tb8_5 using st8 tags(4,5,6.0,true,'7'); + +sql create table tb9_1 using st9 tags(0,1,2.0,true,'3'); +sql create table tb9_2 using st9 tags(1,2,3.0,false,'4'); +sql create table tb9_3 using st9 tags(2,3,4.0,true,'5'); +sql create table tb9_4 using st9 tags(3,4,5.0,false,'6'); +sql create table tb9_5 using st9 tags(4,5,6.0,true,'7'); + +sql create table tba_1 using sta tags(0,1,2.0,true,'3'); +sql create table tba_2 using sta tags(0,1,2.0,true,'3'); +sql create table tba_3 using sta tags(0,1,2.0,true,'3'); +sql create table tba_4 using sta tags(0,1,2.0,true,'3'); +sql create table tba_5 using sta tags(0,1,2.0,true,'3'); + +sql create table tbb_1 using stb tags(0,1,2.0,true,'3'); +sql create table tbb_2 using stb tags(0,1,2.0,true,'3'); +sql create table tbb_3 using stb tags(0,1,2.0,true,'3'); +sql create table tbb_4 using stb tags(0,1,2.0,true,'3'); +sql create table tbb_5 using stb tags(0,1,2.0,true,'3'); + +sql insert into tb0_1 values('2021-03-01 01:00:00.000', 9901,9901.0,'01'); +sql insert into tb0_1 values('2021-03-02 01:00:00.000', 9901,9901.0,'01'); +sql insert into tb0_1 values('2021-03-03 01:00:00.000', 9901,9901.0,'01'); +sql insert into tb0_1 values('2021-03-04 01:00:00.000', 9901,9901.0,'01'); +sql insert into tb0_1 values('2021-03-05 01:00:00.000', 9901,9901.0,'01'); +sql insert into tb0_2 values('2021-03-01 02:00:00.000', 9902,9902.0,'02'); +sql insert into tb0_2 values('2021-03-02 02:00:00.000', 9902,9902.0,'02'); +sql insert into tb0_2 values('2021-03-03 02:00:00.000', 9902,9902.0,'02'); +sql insert into tb0_2 values('2021-03-04 02:00:00.000', 9902,9902.0,'02'); +sql insert into tb0_2 values('2021-03-05 02:00:00.000', 9902,9902.0,'02'); +sql insert into tb0_3 values('2021-03-01 03:00:00.000', 9903,9903.0,'03'); +sql insert into tb0_3 values('2021-03-02 03:00:00.000', 9903,9903.0,'03'); +sql insert into tb0_3 values('2021-03-03 03:00:00.000', 9903,9903.0,'03'); +sql insert into tb0_3 values('2021-03-04 03:00:00.000', 9903,9903.0,'03'); +sql insert into tb0_3 values('2021-03-05 03:00:00.000', 9903,9903.0,'03'); +sql insert into tb0_4 values('2021-03-01 04:00:00.000', 9904,9904.0,'04'); +sql insert into tb0_4 values('2021-03-02 04:00:00.000', 9904,9904.0,'04'); +sql insert into tb0_4 values('2021-03-03 04:00:00.000', 9904,9904.0,'04'); +sql insert into tb0_4 values('2021-03-04 04:00:00.000', 9904,9904.0,'04'); +sql insert into tb0_4 values('2021-03-05 04:00:00.000', 9904,9904.0,'04'); +sql insert into tb0_5 values('2021-03-01 05:00:00.000', 9905,9905.0,'05'); +sql insert into tb0_5 values('2021-03-02 05:00:00.000', 9905,9905.0,'05'); +sql insert into tb0_5 values('2021-03-03 05:00:00.000', 9905,9905.0,'05'); +sql insert into tb0_5 values('2021-03-04 05:00:00.000', 9905,9905.0,'05'); +sql insert into tb0_5 values('2021-03-05 05:00:00.000', 9905,9905.0,'05'); + +sql insert into tb1_1 values('2021-03-01 01:00:00.000', 9911,9911.0,'11'); +sql insert into tb1_1 values('2021-03-02 01:00:00.000', 9911,9911.0,'11'); +sql insert into tb1_1 values('2021-03-03 01:00:00.000', 9911,9911.0,'11'); +sql insert into tb1_1 values('2021-03-04 01:00:00.000', 9911,9911.0,'11'); +sql insert into tb1_1 values('2021-03-05 01:00:00.000', 9911,9911.0,'11'); +sql insert into tb1_2 values('2021-03-01 02:00:00.000', 9912,9912.0,'12'); +sql insert into tb1_2 values('2021-03-02 02:00:00.000', 9912,9912.0,'12'); +sql insert into tb1_2 values('2021-03-03 02:00:00.000', 9912,9912.0,'12'); +sql insert into tb1_2 values('2021-03-04 02:00:00.000', 9912,9912.0,'12'); +sql insert into tb1_2 values('2021-03-05 02:00:00.000', 9912,9912.0,'12'); +sql insert into tb1_3 values('2021-03-01 03:00:00.000', 9913,9913.0,'13'); +sql insert into tb1_3 values('2021-03-02 03:00:00.000', 9913,9913.0,'13'); +sql insert into tb1_3 values('2021-03-03 03:00:00.000', 9913,9913.0,'13'); +sql insert into tb1_3 values('2021-03-04 03:00:00.000', 9913,9913.0,'13'); +sql insert into tb1_3 values('2021-03-05 03:00:00.000', 9913,9913.0,'13'); +sql insert into tb1_4 values('2021-03-01 04:00:00.000', 9914,9914.0,'14'); +sql insert into tb1_4 values('2021-03-02 04:00:00.000', 9914,9914.0,'14'); +sql insert into tb1_4 values('2021-03-03 04:00:00.000', 9914,9914.0,'14'); +sql insert into tb1_4 values('2021-03-04 04:00:00.000', 9914,9914.0,'14'); +sql insert into tb1_4 values('2021-03-05 04:00:00.000', 9914,9914.0,'14'); +sql insert into tb1_5 values('2021-03-01 05:00:00.000', 9915,9915.0,'15'); +sql insert into tb1_5 values('2021-03-02 05:00:00.000', 9915,9915.0,'15'); +sql insert into tb1_5 values('2021-03-03 05:00:00.000', 9915,9915.0,'15'); +sql insert into tb1_5 values('2021-03-04 05:00:00.000', 9915,9915.0,'15'); +sql insert into tb1_5 values('2021-03-05 05:00:00.000', 9915,9915.0,'15'); + +sql insert into tb2_1 values('2021-03-01 01:00:00.000', 9921,9921.0,'21'); +sql insert into tb2_1 values('2021-03-02 01:00:00.000', 9921,9921.0,'21'); +sql insert into tb2_1 values('2021-03-03 01:00:00.000', 9921,9921.0,'21'); +sql insert into tb2_1 values('2021-03-04 01:00:00.000', 9921,9921.0,'21'); +sql insert into tb2_1 values('2021-03-05 01:00:00.000', 9921,9921.0,'21'); +sql insert into tb2_2 values('2021-03-01 02:00:00.000', 9922,9922.0,'22'); +sql insert into tb2_2 values('2021-03-02 02:00:00.000', 9922,9922.0,'22'); +sql insert into tb2_2 values('2021-03-03 02:00:00.000', 9922,9922.0,'22'); +sql insert into tb2_2 values('2021-03-04 02:00:00.000', 9922,9922.0,'22'); +sql insert into tb2_2 values('2021-03-05 02:00:00.000', 9922,9922.0,'22'); +sql insert into tb2_3 values('2021-03-01 03:00:00.000', 9923,9923.0,'23'); +sql insert into tb2_3 values('2021-03-02 03:00:00.000', 9923,9923.0,'23'); +sql insert into tb2_3 values('2021-03-03 03:00:00.000', 9923,9923.0,'23'); +sql insert into tb2_3 values('2021-03-04 03:00:00.000', 9923,9923.0,'23'); +sql insert into tb2_3 values('2021-03-05 03:00:00.000', 9923,9923.0,'23'); +sql insert into tb2_4 values('2021-03-01 04:00:00.000', 9924,9924.0,'24'); +sql insert into tb2_4 values('2021-03-02 04:00:00.000', 9924,9924.0,'24'); +sql insert into tb2_4 values('2021-03-03 04:00:00.000', 9924,9924.0,'24'); +sql insert into tb2_4 values('2021-03-04 04:00:00.000', 9924,9924.0,'24'); +sql insert into tb2_4 values('2021-03-05 04:00:00.000', 9924,9924.0,'24'); +sql insert into tb2_5 values('2021-03-01 05:00:00.000', 9925,9925.0,'25'); +sql insert into tb2_5 values('2021-03-02 05:00:00.000', 9925,9925.0,'25'); +sql insert into tb2_5 values('2021-03-03 05:00:00.000', 9925,9925.0,'25'); +sql insert into tb2_5 values('2021-03-04 05:00:00.000', 9925,9925.0,'25'); +sql insert into tb2_5 values('2021-03-05 05:00:00.000', 9925,9925.0,'25'); + + +sql insert into tb3_1 values('2021-03-01 01:00:00.000', 9931,9931.0,'31'); +sql insert into tb3_1 values('2021-03-02 01:00:00.000', 9931,9931.0,'31'); +sql insert into tb3_1 values('2021-03-03 01:00:00.000', 9931,9931.0,'31'); +sql insert into tb3_1 values('2021-03-04 01:00:00.000', 9931,9931.0,'31'); +sql insert into tb3_1 values('2021-03-05 01:00:00.000', 9931,9931.0,'31'); +sql insert into tb3_2 values('2021-03-01 02:00:00.000', 9932,9932.0,'32'); +sql insert into tb3_2 values('2021-03-02 02:00:00.000', 9932,9932.0,'32'); +sql insert into tb3_2 values('2021-03-03 02:00:00.000', 9932,9932.0,'32'); +sql insert into tb3_2 values('2021-03-04 02:00:00.000', 9932,9932.0,'32'); +sql insert into tb3_2 values('2021-03-05 02:00:00.000', 9932,9932.0,'32'); +sql insert into tb3_3 values('2021-03-01 03:00:00.000', 9933,9933.0,'33'); +sql insert into tb3_3 values('2021-03-02 03:00:00.000', 9933,9933.0,'33'); +sql insert into tb3_3 values('2021-03-03 03:00:00.000', 9933,9933.0,'33'); +sql insert into tb3_3 values('2021-03-04 03:00:00.000', 9933,9933.0,'33'); +sql insert into tb3_3 values('2021-03-05 03:00:00.000', 9933,9933.0,'33'); +sql insert into tb3_4 values('2021-03-01 04:00:00.000', 9934,9934.0,'34'); +sql insert into tb3_4 values('2021-03-02 04:00:00.000', 9934,9934.0,'34'); +sql insert into tb3_4 values('2021-03-03 04:00:00.000', 9934,9934.0,'34'); +sql insert into tb3_4 values('2021-03-04 04:00:00.000', 9934,9934.0,'34'); +sql insert into tb3_4 values('2021-03-05 04:00:00.000', 9934,9934.0,'34'); +sql insert into tb3_5 values('2021-03-01 05:00:00.000', 9935,9935.0,'35'); +sql insert into tb3_5 values('2021-03-02 05:00:00.000', 9935,9935.0,'35'); +sql insert into tb3_5 values('2021-03-03 05:00:00.000', 9935,9935.0,'35'); +sql insert into tb3_5 values('2021-03-04 05:00:00.000', 9935,9935.0,'35'); +sql insert into tb3_5 values('2021-03-05 05:00:00.000', 9935,9935.0,'35'); + + +sql insert into tb4_1 values('2021-03-01 01:00:00.000', 9941,9941.0,'41'); +sql insert into tb4_1 values('2021-03-02 01:00:00.000', 9941,9941.0,'41'); +sql insert into tb4_1 values('2021-03-03 01:00:00.000', 9941,9941.0,'41'); +sql insert into tb4_1 values('2021-03-04 01:00:00.000', 9941,9941.0,'41'); +sql insert into tb4_1 values('2021-03-05 01:00:00.000', 9941,9941.0,'41'); +sql insert into tb4_2 values('2021-03-01 02:00:00.000', 9942,9942.0,'42'); +sql insert into tb4_2 values('2021-03-02 02:00:00.000', 9942,9942.0,'42'); +sql insert into tb4_2 values('2021-03-03 02:00:00.000', 9942,9942.0,'42'); +sql insert into tb4_2 values('2021-03-04 02:00:00.000', 9942,9942.0,'42'); +sql insert into tb4_2 values('2021-03-05 02:00:00.000', 9942,9942.0,'42'); +sql insert into tb4_3 values('2021-03-01 03:00:00.000', 9943,9943.0,'43'); +sql insert into tb4_3 values('2021-03-02 03:00:00.000', 9943,9943.0,'43'); +sql insert into tb4_3 values('2021-03-03 03:00:00.000', 9943,9943.0,'43'); +sql insert into tb4_3 values('2021-03-04 03:00:00.000', 9943,9943.0,'43'); +sql insert into tb4_3 values('2021-03-05 03:00:00.000', 9943,9943.0,'43'); +sql insert into tb4_4 values('2021-03-01 04:00:00.000', 9944,9944.0,'44'); +sql insert into tb4_4 values('2021-03-02 04:00:00.000', 9944,9944.0,'44'); +sql insert into tb4_4 values('2021-03-03 04:00:00.000', 9944,9944.0,'44'); +sql insert into tb4_4 values('2021-03-04 04:00:00.000', 9944,9944.0,'44'); +sql insert into tb4_4 values('2021-03-05 04:00:00.000', 9944,9944.0,'44'); +sql insert into tb4_5 values('2021-03-01 05:00:00.000', 9945,9945.0,'45'); +sql insert into tb4_5 values('2021-03-02 05:00:00.000', 9945,9945.0,'45'); +sql insert into tb4_5 values('2021-03-03 05:00:00.000', 9945,9945.0,'45'); +sql insert into tb4_5 values('2021-03-04 05:00:00.000', 9945,9945.0,'45'); +sql insert into tb4_5 values('2021-03-05 05:00:00.000', 9945,9945.0,'45'); + +sql insert into tb5_1 values('2021-03-01 01:00:00.000', 9951,9951.0,'51'); +sql insert into tb5_1 values('2021-03-02 01:00:00.000', 9951,9951.0,'51'); +sql insert into tb5_1 values('2021-03-03 01:00:00.000', 9951,9951.0,'51'); +sql insert into tb5_1 values('2021-03-04 01:00:00.000', 9951,9951.0,'51'); +sql insert into tb5_1 values('2021-03-05 01:00:00.000', 9951,9951.0,'51'); +sql insert into tb5_2 values('2021-03-01 02:00:00.000', 9952,9952.0,'52'); +sql insert into tb5_2 values('2021-03-02 02:00:00.000', 9952,9952.0,'52'); +sql insert into tb5_2 values('2021-03-03 02:00:00.000', 9952,9952.0,'52'); +sql insert into tb5_2 values('2021-03-04 02:00:00.000', 9952,9952.0,'52'); +sql insert into tb5_2 values('2021-03-05 02:00:00.000', 9952,9952.0,'52'); +sql insert into tb5_3 values('2021-03-01 03:00:00.000', 9953,9953.0,'53'); +sql insert into tb5_3 values('2021-03-02 03:00:00.000', 9953,9953.0,'53'); +sql insert into tb5_3 values('2021-03-03 03:00:00.000', 9953,9953.0,'53'); +sql insert into tb5_3 values('2021-03-04 03:00:00.000', 9953,9953.0,'53'); +sql insert into tb5_3 values('2021-03-05 03:00:00.000', 9953,9953.0,'53'); +sql insert into tb5_4 values('2021-03-01 04:00:00.000', 9954,9954.0,'54'); +sql insert into tb5_4 values('2021-03-02 04:00:00.000', 9954,9954.0,'54'); +sql insert into tb5_4 values('2021-03-03 04:00:00.000', 9954,9954.0,'54'); +sql insert into tb5_4 values('2021-03-04 04:00:00.000', 9954,9954.0,'54'); +sql insert into tb5_4 values('2021-03-05 04:00:00.000', 9954,9954.0,'54'); +sql insert into tb5_5 values('2021-03-01 05:00:00.000', 9955,9955.0,'55'); +sql insert into tb5_5 values('2021-03-02 05:00:00.000', 9955,9955.0,'55'); +sql insert into tb5_5 values('2021-03-03 05:00:00.000', 9955,9955.0,'55'); +sql insert into tb5_5 values('2021-03-04 05:00:00.000', 9955,9955.0,'55'); +sql insert into tb5_5 values('2021-03-05 05:00:00.000', 9955,9955.0,'55'); + +sql insert into tb6_1 values('2021-03-01 01:00:00.000', 9961,9961.0,'61'); +sql insert into tb6_1 values('2021-03-02 01:00:00.000', 9961,9961.0,'61'); +sql insert into tb6_1 values('2021-03-03 01:00:00.000', 9961,9961.0,'61'); +sql insert into tb6_1 values('2021-03-04 01:00:00.000', 9961,9961.0,'61'); +sql insert into tb6_1 values('2021-03-05 01:00:00.000', 9961,9961.0,'61'); +sql insert into tb6_2 values('2021-03-01 02:00:00.000', 9962,9962.0,'62'); +sql insert into tb6_2 values('2021-03-02 02:00:00.000', 9962,9962.0,'62'); +sql insert into tb6_2 values('2021-03-03 02:00:00.000', 9962,9962.0,'62'); +sql insert into tb6_2 values('2021-03-04 02:00:00.000', 9962,9962.0,'62'); +sql insert into tb6_2 values('2021-03-05 02:00:00.000', 9962,9962.0,'62'); +sql insert into tb6_3 values('2021-03-01 03:00:00.000', 9963,9963.0,'63'); +sql insert into tb6_3 values('2021-03-02 03:00:00.000', 9963,9963.0,'63'); +sql insert into tb6_3 values('2021-03-03 03:00:00.000', 9963,9963.0,'63'); +sql insert into tb6_3 values('2021-03-04 03:00:00.000', 9963,9963.0,'63'); +sql insert into tb6_3 values('2021-03-05 03:00:00.000', 9963,9963.0,'63'); +sql insert into tb6_4 values('2021-03-01 04:00:00.000', 9964,9964.0,'64'); +sql insert into tb6_4 values('2021-03-02 04:00:00.000', 9964,9964.0,'64'); +sql insert into tb6_4 values('2021-03-03 04:00:00.000', 9964,9964.0,'64'); +sql insert into tb6_4 values('2021-03-04 04:00:00.000', 9964,9964.0,'64'); +sql insert into tb6_4 values('2021-03-05 04:00:00.000', 9964,9964.0,'64'); +sql insert into tb6_5 values('2021-03-01 05:00:00.000', 9965,9965.0,'65'); +sql insert into tb6_5 values('2021-03-02 05:00:00.000', 9965,9965.0,'65'); +sql insert into tb6_5 values('2021-03-03 05:00:00.000', 9965,9965.0,'65'); +sql insert into tb6_5 values('2021-03-04 05:00:00.000', 9965,9965.0,'65'); +sql insert into tb6_5 values('2021-03-05 05:00:00.000', 9965,9965.0,'65'); + +sql insert into tb7_1 values('2021-03-01 01:00:00.000', 9971,9971.0,'71'); +sql insert into tb7_1 values('2021-03-02 01:00:00.000', 9971,9971.0,'71'); +sql insert into tb7_1 values('2021-03-03 01:00:00.000', 9971,9971.0,'71'); +sql insert into tb7_1 values('2021-03-04 01:00:00.000', 9971,9971.0,'71'); +sql insert into tb7_1 values('2021-03-05 01:00:00.000', 9971,9971.0,'71'); +sql insert into tb7_2 values('2021-03-01 02:00:00.000', 9972,9972.0,'72'); +sql insert into tb7_2 values('2021-03-02 02:00:00.000', 9972,9972.0,'72'); +sql insert into tb7_2 values('2021-03-03 02:00:00.000', 9972,9972.0,'72'); +sql insert into tb7_2 values('2021-03-04 02:00:00.000', 9972,9972.0,'72'); +sql insert into tb7_2 values('2021-03-05 02:00:00.000', 9972,9972.0,'72'); +sql insert into tb7_3 values('2021-03-01 03:00:00.000', 9973,9973.0,'73'); +sql insert into tb7_3 values('2021-03-02 03:00:00.000', 9973,9973.0,'73'); +sql insert into tb7_3 values('2021-03-03 03:00:00.000', 9973,9973.0,'73'); +sql insert into tb7_3 values('2021-03-04 03:00:00.000', 9973,9973.0,'73'); +sql insert into tb7_3 values('2021-03-05 03:00:00.000', 9973,9973.0,'73'); +sql insert into tb7_4 values('2021-03-01 04:00:00.000', 9974,9974.0,'74'); +sql insert into tb7_4 values('2021-03-02 04:00:00.000', 9974,9974.0,'74'); +sql insert into tb7_4 values('2021-03-03 04:00:00.000', 9974,9974.0,'74'); +sql insert into tb7_4 values('2021-03-04 04:00:00.000', 9974,9974.0,'74'); +sql insert into tb7_4 values('2021-03-05 04:00:00.000', 9974,9974.0,'74'); +sql insert into tb7_5 values('2021-03-01 05:00:00.000', 9975,9975.0,'75'); +sql insert into tb7_5 values('2021-03-02 05:00:00.000', 9975,9975.0,'75'); +sql insert into tb7_5 values('2021-03-03 05:00:00.000', 9975,9975.0,'75'); +sql insert into tb7_5 values('2021-03-04 05:00:00.000', 9975,9975.0,'75'); +sql insert into tb7_5 values('2021-03-05 05:00:00.000', 9975,9975.0,'75'); + +sql insert into tb8_1 values('2021-03-01 01:00:00.000', 9981,9981.0,'81'); +sql insert into tb8_1 values('2021-03-02 01:00:00.000', 9981,9981.0,'81'); +sql insert into tb8_1 values('2021-03-03 01:00:00.000', 9981,9981.0,'81'); +sql insert into tb8_1 values('2021-03-04 01:00:00.000', 9981,9981.0,'81'); +sql insert into tb8_1 values('2021-03-05 01:00:00.000', 9981,9981.0,'81'); +sql insert into tb8_2 values('2021-03-01 02:00:00.000', 9982,9982.0,'82'); +sql insert into tb8_2 values('2021-03-02 02:00:00.000', 9982,9982.0,'82'); +sql insert into tb8_2 values('2021-03-03 02:00:00.000', 9982,9982.0,'82'); +sql insert into tb8_2 values('2021-03-04 02:00:00.000', 9982,9982.0,'82'); +sql insert into tb8_2 values('2021-03-05 02:00:00.000', 9982,9982.0,'82'); +sql insert into tb8_3 values('2021-03-01 03:00:00.000', 9983,9983.0,'83'); +sql insert into tb8_3 values('2021-03-02 03:00:00.000', 9983,9983.0,'83'); +sql insert into tb8_3 values('2021-03-03 03:00:00.000', 9983,9983.0,'83'); +sql insert into tb8_3 values('2021-03-04 03:00:00.000', 9983,9983.0,'83'); +sql insert into tb8_3 values('2021-03-05 03:00:00.000', 9983,9983.0,'83'); +sql insert into tb8_4 values('2021-03-01 04:00:00.000', 9984,9984.0,'84'); +sql insert into tb8_4 values('2021-03-02 04:00:00.000', 9984,9984.0,'84'); +sql insert into tb8_4 values('2021-03-03 04:00:00.000', 9984,9984.0,'84'); +sql insert into tb8_4 values('2021-03-04 04:00:00.000', 9984,9984.0,'84'); +sql insert into tb8_4 values('2021-03-05 04:00:00.000', 9984,9984.0,'84'); +sql insert into tb8_5 values('2021-03-01 05:00:00.000', 9985,9985.0,'85'); +sql insert into tb8_5 values('2021-03-02 05:00:00.000', 9985,9985.0,'85'); +sql insert into tb8_5 values('2021-03-03 05:00:00.000', 9985,9985.0,'85'); +sql insert into tb8_5 values('2021-03-04 05:00:00.000', 9985,9985.0,'85'); +sql insert into tb8_5 values('2021-03-05 05:00:00.000', 9985,9985.0,'85'); + +sql insert into tb9_1 values('2021-03-01 01:00:00.000', 9991,9991.0,'91'); +sql insert into tb9_1 values('2021-03-02 01:00:00.000', 9991,9991.0,'91'); +sql insert into tb9_1 values('2021-03-03 01:00:00.000', 9991,9991.0,'91'); +sql insert into tb9_1 values('2021-03-04 01:00:00.000', 9991,9991.0,'91'); +sql insert into tb9_1 values('2021-03-05 01:00:00.000', 9991,9991.0,'91'); +sql insert into tb9_2 values('2021-03-01 02:00:00.000', 9992,9992.0,'92'); +sql insert into tb9_2 values('2021-03-02 02:00:00.000', 9992,9992.0,'92'); +sql insert into tb9_2 values('2021-03-03 02:00:00.000', 9992,9992.0,'92'); +sql insert into tb9_2 values('2021-03-04 02:00:00.000', 9992,9992.0,'92'); +sql insert into tb9_2 values('2021-03-05 02:00:00.000', 9992,9992.0,'92'); +sql insert into tb9_3 values('2021-03-01 03:00:00.000', 9993,9993.0,'93'); +sql insert into tb9_3 values('2021-03-02 03:00:00.000', 9993,9993.0,'93'); +sql insert into tb9_3 values('2021-03-03 03:00:00.000', 9993,9993.0,'93'); +sql insert into tb9_3 values('2021-03-04 03:00:00.000', 9993,9993.0,'93'); +sql insert into tb9_3 values('2021-03-05 03:00:00.000', 9993,9993.0,'93'); +sql insert into tb9_4 values('2021-03-01 04:00:00.000', 9994,9994.0,'94'); +sql insert into tb9_4 values('2021-03-02 04:00:00.000', 9994,9994.0,'94'); +sql insert into tb9_4 values('2021-03-03 04:00:00.000', 9994,9994.0,'94'); +sql insert into tb9_4 values('2021-03-04 04:00:00.000', 9994,9994.0,'94'); +sql insert into tb9_4 values('2021-03-05 04:00:00.000', 9994,9994.0,'94'); +sql insert into tb9_5 values('2021-03-01 05:00:00.000', 9995,9995.0,'95'); +sql insert into tb9_5 values('2021-03-02 05:00:00.000', 9995,9995.0,'95'); +sql insert into tb9_5 values('2021-03-03 05:00:00.000', 9995,9995.0,'95'); +sql insert into tb9_5 values('2021-03-04 05:00:00.000', 9995,9995.0,'95'); +sql insert into tb9_5 values('2021-03-05 05:00:00.000', 9995,9995.0,'95'); + +sql insert into tba_1 values('2021-03-01 01:00:00.000', 99101,99101.0,'a1'); +sql insert into tba_1 values('2021-03-02 01:00:00.000', 99101,99101.0,'a1'); +sql insert into tba_1 values('2021-03-03 01:00:00.000', 99101,99101.0,'a1'); +sql insert into tba_1 values('2021-03-04 01:00:00.000', 99101,99101.0,'a1'); +sql insert into tba_1 values('2021-03-05 01:00:00.000', 99101,99101.0,'a1'); +sql insert into tba_2 values('2021-03-01 02:00:00.000', 99102,99102.0,'a2'); +sql insert into tba_2 values('2021-03-02 02:00:00.000', 99102,99102.0,'a2'); +sql insert into tba_2 values('2021-03-03 02:00:00.000', 99102,99102.0,'a2'); +sql insert into tba_2 values('2021-03-04 02:00:00.000', 99102,99102.0,'a2'); +sql insert into tba_2 values('2021-03-05 02:00:00.000', 99102,99102.0,'a2'); +sql insert into tba_3 values('2021-03-01 03:00:00.000', 99103,99103.0,'a3'); +sql insert into tba_3 values('2021-03-02 03:00:00.000', 99103,99103.0,'a3'); +sql insert into tba_3 values('2021-03-03 03:00:00.000', 99103,99103.0,'a3'); +sql insert into tba_3 values('2021-03-04 03:00:00.000', 99103,99103.0,'a3'); +sql insert into tba_3 values('2021-03-05 03:00:00.000', 99103,99103.0,'a3'); +sql insert into tba_4 values('2021-03-01 04:00:00.000', 99104,99104.0,'a4'); +sql insert into tba_4 values('2021-03-02 04:00:00.000', 99104,99104.0,'a4'); +sql insert into tba_4 values('2021-03-03 04:00:00.000', 99104,99104.0,'a4'); +sql insert into tba_4 values('2021-03-04 04:00:00.000', 99104,99104.0,'a4'); +sql insert into tba_4 values('2021-03-05 04:00:00.000', 99104,99104.0,'a4'); +sql insert into tba_5 values('2021-03-01 05:00:00.000', 99105,99105.0,'a5'); +sql insert into tba_5 values('2021-03-02 05:00:00.000', 99105,99105.0,'a5'); +sql insert into tba_5 values('2021-03-03 05:00:00.000', 99105,99105.0,'a5'); +sql insert into tba_5 values('2021-03-04 05:00:00.000', 99105,99105.0,'a5'); +sql insert into tba_5 values('2021-03-05 05:00:00.000', 99105,99105.0,'a5'); + +sql insert into tbb_1 values('2021-03-01 01:00:00.000', 99111,99111.0,'b1'); +sql insert into tbb_1 values('2021-03-02 01:00:00.000', 99111,99111.0,'b1'); +sql insert into tbb_1 values('2021-03-03 01:00:00.000', 99111,99111.0,'b1'); +sql insert into tbb_1 values('2021-03-04 01:00:00.000', 99111,99111.0,'b1'); +sql insert into tbb_1 values('2021-03-05 01:00:00.000', 99111,99111.0,'b1'); +sql insert into tbb_2 values('2021-03-01 02:00:00.000', 99112,99112.0,'b2'); +sql insert into tbb_2 values('2021-03-02 02:00:00.000', 99112,99112.0,'b2'); +sql insert into tbb_2 values('2021-03-03 02:00:00.000', 99112,99112.0,'b2'); +sql insert into tbb_2 values('2021-03-04 02:00:00.000', 99112,99112.0,'b2'); +sql insert into tbb_2 values('2021-03-05 02:00:00.000', 99112,99112.0,'b2'); +sql insert into tbb_3 values('2021-03-01 03:00:00.000', 99113,99113.0,'b3'); +sql insert into tbb_3 values('2021-03-02 03:00:00.000', 99113,99113.0,'b3'); +sql insert into tbb_3 values('2021-03-03 03:00:00.000', 99113,99113.0,'b3'); +sql insert into tbb_3 values('2021-03-04 03:00:00.000', 99113,99113.0,'b3'); +sql insert into tbb_3 values('2021-03-05 03:00:00.000', 99113,99113.0,'b3'); +sql insert into tbb_4 values('2021-03-01 04:00:00.000', 99114,99114.0,'b4'); +sql insert into tbb_4 values('2021-03-02 04:00:00.000', 99114,99114.0,'b4'); +sql insert into tbb_4 values('2021-03-03 04:00:00.000', 99114,99114.0,'b4'); +sql insert into tbb_4 values('2021-03-04 04:00:00.000', 99114,99114.0,'b4'); +sql insert into tbb_4 values('2021-03-05 04:00:00.000', 99114,99114.0,'b4'); +sql insert into tbb_5 values('2021-03-01 05:00:00.000', 99115,99115.0,'b5'); +sql insert into tbb_5 values('2021-03-02 05:00:00.000', 99115,99115.0,'b5'); +sql insert into tbb_5 values('2021-03-03 05:00:00.000', 99115,99115.0,'b5'); +sql insert into tbb_5 values('2021-03-04 05:00:00.000', 99115,99115.0,'b5'); +sql insert into tbb_5 values('2021-03-05 05:00:00.000', 99115,99115.0,'b5'); + + +sql select * from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1; + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + print $data07 + return -1 +endi +if $data08 != 3 then + return -1 +endi +if $data09 != @21-03-01 01:00:00.000@ then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != 0 then + return -1 +endi +if $data15 != 1 then + return -1 +endi +if $data16 != 2.000000000 then + return -1 +endi +if $data17 != 1 then + return -1 +endi +if $data18 != 3 then + return -1 +endi +if $data19 != @21-03-02 01:00:00.000@ then + return -1 +endi + + + +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != 9901 then + return -1 +endi +if $data22 != 9901.000000000 then + return -1 +endi +if $data23 != 01 then + return -1 +endi +if $data24 != 0 then + return -1 +endi +if $data25 != 1 then + return -1 +endi +if $data26 != 2.000000000 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != 3 then + return -1 +endi +if $data29 != @21-03-03 01:00:00.000@ then + return -1 +endi + + +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != 9901 then + return -1 +endi +if $data32 != 9901.000000000 then + return -1 +endi +if $data33 != 01 then + return -1 +endi +if $data34 != 0 then + return -1 +endi +if $data35 != 1 then + return -1 +endi +if $data36 != 2.000000000 then + return -1 +endi +if $data37 != 1 then + return -1 +endi +if $data38 != 3 then + return -1 +endi +if $data39 != @21-03-04 01:00:00.000@ then + return -1 +endi + + + + +sql select * from st0, st1 where st0.ts=st1.ts and st0.id2=st1.id2; + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + print $data07 + return -1 +endi +if $data08 != 3 then + return -1 +endi +if $data09 != @21-03-01 01:00:00.000@ then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != 0 then + return -1 +endi +if $data15 != 1 then + return -1 +endi +if $data16 != 2.000000000 then + return -1 +endi +if $data17 != 1 then + return -1 +endi +if $data18 != 3 then + return -1 +endi +if $data19 != @21-03-02 01:00:00.000@ then + return -1 +endi + + + +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != 9901 then + return -1 +endi +if $data22 != 9901.000000000 then + return -1 +endi +if $data23 != 01 then + return -1 +endi +if $data24 != 0 then + return -1 +endi +if $data25 != 1 then + return -1 +endi +if $data26 != 2.000000000 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != 3 then + return -1 +endi +if $data29 != @21-03-03 01:00:00.000@ then + return -1 +endi + + +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != 9901 then + return -1 +endi +if $data32 != 9901.000000000 then + return -1 +endi +if $data33 != 01 then + return -1 +endi +if $data34 != 0 then + return -1 +endi +if $data35 != 1 then + return -1 +endi +if $data36 != 2.000000000 then + return -1 +endi +if $data37 != 1 then + return -1 +endi +if $data38 != 3 then + return -1 +endi +if $data39 != @21-03-04 01:00:00.000@ then + return -1 +endi + + + + +sql select * from st0, st1 where st0.id3=st1.id3 and st1.ts=st0.ts; + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + print $data07 + return -1 +endi +if $data08 != 3 then + return -1 +endi +if $data09 != @21-03-01 01:00:00.000@ then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != 0 then + return -1 +endi +if $data15 != 1 then + return -1 +endi +if $data16 != 2.000000000 then + return -1 +endi +if $data17 != 1 then + return -1 +endi +if $data18 != 3 then + return -1 +endi +if $data19 != @21-03-02 01:00:00.000@ then + return -1 +endi + + + +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != 9901 then + return -1 +endi +if $data22 != 9901.000000000 then + return -1 +endi +if $data23 != 01 then + return -1 +endi +if $data24 != 0 then + return -1 +endi +if $data25 != 1 then + return -1 +endi +if $data26 != 2.000000000 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != 3 then + return -1 +endi +if $data29 != @21-03-03 01:00:00.000@ then + return -1 +endi + + +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != 9901 then + return -1 +endi +if $data32 != 9901.000000000 then + return -1 +endi +if $data33 != 01 then + return -1 +endi +if $data34 != 0 then + return -1 +endi +if $data35 != 1 then + return -1 +endi +if $data36 != 2.000000000 then + return -1 +endi +if $data37 != 1 then + return -1 +endi +if $data38 != 3 then + return -1 +endi +if $data39 != @21-03-04 01:00:00.000@ then + return -1 +endi + + + + + +sql select * from st0, st1 where st1.id5=st0.id5 and st0.ts=st1.ts; + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + print $data07 + return -1 +endi +if $data08 != 3 then + return -1 +endi +if $data09 != @21-03-01 01:00:00.000@ then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != 0 then + return -1 +endi +if $data15 != 1 then + return -1 +endi +if $data16 != 2.000000000 then + return -1 +endi +if $data17 != 1 then + return -1 +endi +if $data18 != 3 then + return -1 +endi +if $data19 != @21-03-02 01:00:00.000@ then + return -1 +endi + + + +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != 9901 then + return -1 +endi +if $data22 != 9901.000000000 then + return -1 +endi +if $data23 != 01 then + return -1 +endi +if $data24 != 0 then + return -1 +endi +if $data25 != 1 then + return -1 +endi +if $data26 != 2.000000000 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != 3 then + return -1 +endi +if $data29 != @21-03-03 01:00:00.000@ then + return -1 +endi + + +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != 9901 then + return -1 +endi +if $data32 != 9901.000000000 then + return -1 +endi +if $data33 != 01 then + return -1 +endi +if $data34 != 0 then + return -1 +endi +if $data35 != 1 then + return -1 +endi +if $data36 != 2.000000000 then + return -1 +endi +if $data37 != 1 then + return -1 +endi +if $data38 != 3 then + return -1 +endi +if $data39 != @21-03-04 01:00:00.000@ then + return -1 +endi + + + + +sql select st0.* from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1; + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + print $data07 + return -1 +endi +if $data08 != 3 then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != 0 then + return -1 +endi +if $data15 != 1 then + return -1 +endi +if $data16 != 2.000000000 then + return -1 +endi +if $data17 != 1 then + return -1 +endi +if $data18 != 3 then + return -1 +endi + + + +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != 9901 then + return -1 +endi +if $data22 != 9901.000000000 then + return -1 +endi +if $data23 != 01 then + return -1 +endi +if $data24 != 0 then + return -1 +endi +if $data25 != 1 then + return -1 +endi +if $data26 != 2.000000000 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != 3 then + return -1 +endi + + +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != 9901 then + return -1 +endi +if $data32 != 9901.000000000 then + return -1 +endi +if $data33 != 01 then + return -1 +endi +if $data34 != 0 then + return -1 +endi +if $data35 != 1 then + return -1 +endi +if $data36 != 2.000000000 then + return -1 +endi +if $data37 != 1 then + return -1 +endi +if $data38 != 3 then + return -1 +endi + + + + +sql select st0.* from st0, st1 where st0.ts=st1.ts and st1.id2=st0.id2; + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + print $data07 + return -1 +endi +if $data08 != 3 then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != 0 then + return -1 +endi +if $data15 != 1 then + return -1 +endi +if $data16 != 2.000000000 then + return -1 +endi +if $data17 != 1 then + return -1 +endi +if $data18 != 3 then + return -1 +endi + + + +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != 9901 then + return -1 +endi +if $data22 != 9901.000000000 then + return -1 +endi +if $data23 != 01 then + return -1 +endi +if $data24 != 0 then + return -1 +endi +if $data25 != 1 then + return -1 +endi +if $data26 != 2.000000000 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != 3 then + return -1 +endi + + +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != 9901 then + return -1 +endi +if $data32 != 9901.000000000 then + return -1 +endi +if $data33 != 01 then + return -1 +endi +if $data34 != 0 then + return -1 +endi +if $data35 != 1 then + return -1 +endi +if $data36 != 2.000000000 then + return -1 +endi +if $data37 != 1 then + return -1 +endi +if $data38 != 3 then + return -1 +endi + + + + + +sql select st0.* from st0, st1 where st0.id3=st1.id3 and st1.ts=st0.ts; + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + print $data07 + return -1 +endi +if $data08 != 3 then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != 0 then + return -1 +endi +if $data15 != 1 then + return -1 +endi +if $data16 != 2.000000000 then + return -1 +endi +if $data17 != 1 then + return -1 +endi +if $data18 != 3 then + return -1 +endi + + + +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != 9901 then + return -1 +endi +if $data22 != 9901.000000000 then + return -1 +endi +if $data23 != 01 then + return -1 +endi +if $data24 != 0 then + return -1 +endi +if $data25 != 1 then + return -1 +endi +if $data26 != 2.000000000 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != 3 then + return -1 +endi + + +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != 9901 then + return -1 +endi +if $data32 != 9901.000000000 then + return -1 +endi +if $data33 != 01 then + return -1 +endi +if $data34 != 0 then + return -1 +endi +if $data35 != 1 then + return -1 +endi +if $data36 != 2.000000000 then + return -1 +endi +if $data37 != 1 then + return -1 +endi +if $data38 != 3 then + return -1 +endi + + + + +sql select st1.* from st0, st1 where st1.id5=st0.id5 and st0.ts=st1.ts; + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9911 then + return -1 +endi +if $data02 != 9911.000000000 then + return -1 +endi +if $data03 != 11 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + print $data07 + return -1 +endi +if $data08 != 3 then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9911 then + return -1 +endi +if $data12 != 9911.000000000 then + return -1 +endi +if $data13 != 11 then + return -1 +endi +if $data14 != 0 then + return -1 +endi +if $data15 != 1 then + return -1 +endi +if $data16 != 2.000000000 then + return -1 +endi +if $data17 != 1 then + return -1 +endi +if $data18 != 3 then + return -1 +endi + + + +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != 9911 then + return -1 +endi +if $data22 != 9911.000000000 then + return -1 +endi +if $data23 != 11 then + return -1 +endi +if $data24 != 0 then + return -1 +endi +if $data25 != 1 then + return -1 +endi +if $data26 != 2.000000000 then + return -1 +endi +if $data27 != 1 then + return -1 +endi +if $data28 != 3 then + return -1 +endi + + +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != 9911 then + return -1 +endi +if $data32 != 9911.000000000 then + return -1 +endi +if $data33 != 11 then + return -1 +endi +if $data34 != 0 then + return -1 +endi +if $data35 != 1 then + return -1 +endi +if $data36 != 2.000000000 then + return -1 +endi +if $data37 != 1 then + return -1 +endi +if $data38 != 3 then + return -1 +endi + + +sql select st0.f1,st1.f1 from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1; + +if $rows != 25 then + return -1 +endi + +if $data00 != 9901 then + return -1 +endi +if $data01 != 9911 then + return -1 +endi +if $data10 != 9901 then + return -1 +endi +if $data11 != 9911 then + return -1 +endi +if $data20 != 9901 then + return -1 +endi +if $data21 != 9911 then + return -1 +endi +if $data30 != 9901 then + return -1 +endi +if $data31 != 9911 then + return -1 +endi + + + + +sql select st0.ts,st1.ts from st0, st1 where st0.ts=st1.ts and st1.id2=st0.id2; + + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data20 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data21 != @21-03-03 01:00:00.000@ then + return -1 +endi +if $data30 != @21-03-04 01:00:00.000@ then + return -1 +endi +if $data31 != @21-03-04 01:00:00.000@ then + return -1 +endi + + + +sql select st1.ts,st0.ts,st0.id3,st1.id3,st0.f3,st1.f3 from st0, st1 where st0.id3=st1.id3 and st1.ts=st0.ts; + + +if $rows != 25 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data02 != 2.000000000 then + return -1 +endi +if $data03 != 2.000000000 then + return -1 +endi +if $data04 != 01 then + return -1 +endi +if $data05 != 11 then + return -1 +endi + +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data02 != 2.000000000 then + return -1 +endi +if $data03 != 2.000000000 then + return -1 +endi +if $data04 != 01 then + return -1 +endi +if $data05 != 11 then + return -1 +endi + + + +sql select st0.ts,st0.f2,st1.f3,st1.f2,st0.f3 from st0, st1 where st1.id5=st0.id5 and st0.ts=st1.ts; +if $rows != 25 then + return -1 +endi +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901.000000000 then + return -1 +endi +if $data02 != 11 then + return -1 +endi +if $data03 != 9911.000000000 then + return -1 +endi +if $data04 != 01 then + return -1 +endi +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901.000000000 then + return -1 +endi +if $data12 != 11 then + return -1 +endi +if $data13 != 9911.000000000 then + return -1 +endi +if $data14 != 01 then + return -1 +endi + + + + +sql select last(*) from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 interval(10a); +if $rows != 25 then + return -1 +endi +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data02 != 9901 then + return -1 +endi +if $data03 != 9901.000000000 then + return -1 +endi +if $data04 != 01 then + return -1 +endi +if $data05 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data06 != 9911 then + return -1 +endi +if $data07 != 9911.000000000 then + return -1 +endi +if $data08 != 11 then + return -1 +endi +if $data10 != @21-03-01 02:00:00.000@ then + return -1 +endi +if $data11 != @21-03-01 02:00:00.000@ then + return -1 +endi +if $data12 != 9902 then + return -1 +endi +if $data13 != 9902.000000000 then + return -1 +endi +if $data14 != 02 then + return -1 +endi +if $data15 != @21-03-01 02:00:00.000@ then + return -1 +endi +if $data16 != 9912 then + return -1 +endi +if $data17 != 9912.000000000 then + return -1 +endi +if $data18 != 12 then + return -1 +endi + + + +sql select last(*) from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 interval(1d) sliding(1d); +if $rows != 5 then + return -1 +endi +if $data00 != @21-03-01 00:00:00.000@ then + return -1 +endi +if $data01 != @21-03-01 05:00:00.000@ then + return -1 +endi +if $data02 != 9905 then + return -1 +endi +if $data03 != 9905.000000000 then + return -1 +endi +if $data04 != 05 then + return -1 +endi +if $data05 != @21-03-01 05:00:00.000@ then + return -1 +endi +if $data06 != 9915 then + return -1 +endi +if $data07 != 9915.000000000 then + return -1 +endi +if $data08 != 15 then + return -1 +endi +if $data10 != @21-03-02 00:00:00.000@ then + return -1 +endi +if $data11 != @21-03-02 05:00:00.000@ then + return -1 +endi +if $data12 != 9905 then + return -1 +endi +if $data13 != 9905.000000000 then + return -1 +endi +if $data14 != 05 then + return -1 +endi +if $data15 != @21-03-02 05:00:00.000@ then + return -1 +endi +if $data16 != 9915 then + return -1 +endi +if $data17 != 9915.000000000 then + return -1 +endi +if $data18 != 15 then + return -1 +endi + + + +sql select st0.*,st1.* from st0, st1 where st1.id1=st0.id1 and st0.ts=st1.ts and st1.ts=st0.ts and st0.id1=st1.id1; +if $rows != 25 then + return -1 +endi +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != 3 then + return -1 +endi +if $data09 != @21-03-01 01:00:00.000@ then + return -1 +endi + + + + + +sql select st0.ts,* from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 order by st0.ts; + +if $rows != 25 then + return -1 +endi +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data02 != 9901 then + return -1 +endi +if $data03 != 9901.000000000 then + return -1 +endi +if $data04 != 01 then + return -1 +endi +if $data05 != 0 then + return -1 +endi +if $data06 != 1 then + return -1 +endi +if $data07 != 2.000000000 then + return -1 +endi +if $data08 != 1 then + return -1 +endi +if $data09 != 3 then + return -1 +endi + + + + + +sql select st0.*,st1.* from st0, st1 where st1.id1=st0.id1 and st0.ts=st1.ts and st1.ts=st0.ts and st0.id1=st1.id1 order by st0.ts limit 5 offset 5 +if $rows != 5 then + return -1 +endi +if $data00 != @21-03-01 01:00:00.000@ then + print $data00 + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != 3 then + return -1 +endi +if $data09 != @21-03-01 01:00:00.000@ then + return -1 +endi + + + +sql select top(st1.f1, 5) from st0, st1 where st1.id1=st0.id1 and st0.ts=st1.ts and st1.ts=st0.ts and st0.id1=st1.id1; +if $rows != 5 then + return -1 +endi +if $data00 != @21-03-01 05:00:00.000@ then + return -1 +endi +if $data01 != 9915 then + return -1 +endi +if $data10 != @21-03-02 05:00:00.000@ then + return -1 +endi +if $data11 != 9915 then + return -1 +endi +if $data20 != @21-03-03 05:00:00.000@ then + return -1 +endi +if $data21 != 9915 then + return -1 +endi +if $data30 != @21-03-04 05:00:00.000@ then + return -1 +endi +if $data31 != 9915 then + return -1 +endi +if $data40 != @21-03-05 05:00:00.000@ then + return -1 +endi +if $data41 != 9915 then + return -1 +endi + + + + + +sql select top(st0.f1,5) from st0, st1 where st1.id1=st0.id1 and st0.ts=st1.ts and st1.ts=st0.ts and st0.id1=st1.id1; +if $rows != 5 then + return -1 +endi +if $data00 != @21-03-01 05:00:00.000@ then + return -1 +endi +if $data01 != 9905 then + return -1 +endi +if $data10 != @21-03-02 05:00:00.000@ then + return -1 +endi +if $data11 != 9905 then + return -1 +endi +if $data20 != @21-03-03 05:00:00.000@ then + return -1 +endi +if $data21 != 9905 then + return -1 +endi +if $data30 != @21-03-04 05:00:00.000@ then + return -1 +endi +if $data31 != 9905 then + return -1 +endi +if $data40 != @21-03-05 05:00:00.000@ then + return -1 +endi +if $data41 != 9905 then + return -1 +endi + + +#sql select st0.*,st1.*,st2.*,st3.* from st3,st2,st1,st0 where st0.id1=st3.id1 and st3.ts=st2.ts and st2.id1=st1.id1 and st1.ts=st0.ts; +#sql select st0.*,st1.*,st2.*,st3.* from st3,st2,st1,st0 where st0.id1=st3.id1 and st3.ts=st2.ts and st2.id1=st1.id1 and st1.ts=st0.ts and st0.id1=st2.id1 and st1.ts=st2.ts; +#if $rows != 25 then +# print $rows +# return -1 +#endi +#if $data00 != @21-03-01 01:00:00.000@ then +# return -1 +#endi +#if $data01 != 9901 then +# return -1 +#endi +#if $data02 != 9901.000000000 then +# return -1 +#endi +#if $data03 != 01 then +# return -1 +#endi +#if $data04 != 0 then +# return -1 +#endi +#if $data05 != 1 then +# return -1 +#endi +#if $data06 != 2.000000000 then +# return -1 +#endi +#if $data07 != 1 then +# return -1 +#endi +#if $data08 != 3 then +# return -1 +#endi +#if $data09 != @21-03-01 01:00:00.000@ then +# return -1 +#endi + + + +#sql select st0.*,st1.*,st2.*,st3.* from st3,st2,st1,st0 where st0.id1=st1.id1 and st1.ts=st0.ts and st2.id1=st3.id1 and st3.ts=st2.ts; +#sql select st0.*,st1.*,st2.*,st3.* from st3,st2,st1,st0 where st0.id1=st1.id1 and st1.ts=st0.ts and st2.id1=st3.id1 and st3.ts=st2.ts and st0.id1=st2.id1 and st0.ts=st2.ts; +#if $rows != 25 then +# return -1 +#endi +#if $data00 != @21-03-01 01:00:00.000@ then +# return -1 +#endi +#if $data01 != 9901 then +# return -1 +#endi +#if $data02 != 9901.000000000 then +# return -1 +#endi +#if $data03 != 01 then +# return -1 +#endi +#if $data04 != 0 then +# return -1 +#endi +#if $data05 != 1 then +# return -1 +#endi +#if $data06 != 2.000000000 then +# return -1 +#endi +#if $data07 != 1 then +# return -1 +#endi +#if $data08 != 3 then +# return -1 +#endi +#if $data09 != @21-03-01 01:00:00.000@ then +# return -1 +#endi + + + +sql select st0.*,st1.*,st2.*,st3.*,st4.*,st5.*,st6.*,st7.*,st8.*,st9.* from st0,st1,st2,st3,st4,st5,st6,st7,st8,st9 where st0.ts=st2.ts and st0.ts=st4.ts and st0.ts=st6.ts and st0.ts=st8.ts and st1.ts=st3.ts and st3.ts=st5.ts and st5.ts=st7.ts and st7.ts=st9.ts and st0.ts=st1.ts and st0.id1=st2.id1 and st0.id1=st4.id1 and st0.id1=st6.id1 and st0.id1=st8.id1 and st1.id1=st3.id1 and st3.id1=st5.id1 and st5.id1=st7.id1 and st7.id1=st9.id1 and st0.id1=st1.id1; +if $rows != 25 then + return -1 +endi +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != 0 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 2.000000000 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != 3 then + return -1 +endi +if $data09 != @21-03-01 01:00:00.000@ then + return -1 +endi + + + +sql select tb0_1.*, tb1_1.* from tb0_1, tb1_1 where tb0_1.ts=tb1_1.ts; +if $rows != 5 then + return -1 +endi +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data05 != 9911 then + return -1 +endi +if $data06 != 9911.000000000 then + return -1 +endi +if $data07 != 11 then + return -1 +endi +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data15 != 9911 then + return -1 +endi +if $data16 != 9911.000000000 then + return -1 +endi +if $data17 != 11 then + return -1 +endi + + + +sql select tb0_1.*, tb1_1.* from tb0_1, tb1_1 where tb0_1.ts=tb1_1.ts and tb0_1.id1=tb1_1.id1; +if $rows != 5 then + return -1 +endi +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data05 != 9911 then + return -1 +endi +if $data06 != 9911.000000000 then + return -1 +endi +if $data07 != 11 then + return -1 +endi +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data15 != 9911 then + return -1 +endi +if $data16 != 9911.000000000 then + return -1 +endi +if $data17 != 11 then + return -1 +endi + + +sql select tb0_1.*, tb1_2.*,tb2_3.*,tb3_4.*,tb4_5.* from tb0_1, tb1_2, tb2_3, tb3_4, tb4_5 where tb0_1.ts=tb1_2.ts and tb0_1.ts=tb2_3.ts and tb0_1.ts=tb3_4.ts and tb0_1.ts=tb4_5.ts; +if $rows != 0 then + return -1 +endi + + + +sql select tb0_1.*, tb1_1.*,tb2_1.*,tb3_1.*,tb4_1.* from tb0_1, tb1_1, tb2_1, tb3_1, tb4_1 where tb0_1.ts=tb1_1.ts and tb0_1.ts=tb2_1.ts and tb0_1.ts=tb3_1.ts and tb0_1.ts=tb4_1.ts; + +if $rows != 5 then + return -1 +endi + +if $data00 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data01 != 9901 then + return -1 +endi +if $data02 != 9901.000000000 then + return -1 +endi +if $data03 != 01 then + return -1 +endi +if $data04 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data05 != 9911 then + return -1 +endi +if $data06 != 9911.000000000 then + return -1 +endi +if $data07 != 11 then + return -1 +endi +if $data08 != @21-03-01 01:00:00.000@ then + return -1 +endi +if $data09 != 9921 then + return -1 +endi +if $data10 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data11 != 9901 then + return -1 +endi +if $data12 != 9901.000000000 then + return -1 +endi +if $data13 != 01 then + return -1 +endi +if $data14 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data15 != 9911 then + return -1 +endi +if $data16 != 9911.000000000 then + return -1 +endi +if $data17 != 11 then + return -1 +endi +if $data18 != @21-03-02 01:00:00.000@ then + return -1 +endi +if $data19 != 9921 then + return -1 +endi + + + +sql select tb0_5.*, tb1_5.*,tb2_5.*,tb3_5.*,tb4_5.*,tb5_5.*, tb6_5.*,tb7_5.*,tb8_5.*,tb9_5.* from tb0_5, tb1_5, tb2_5, tb3_5, tb4_5,tb5_5, tb6_5, tb7_5, tb8_5, tb9_5 where tb9_5.ts=tb8_5.ts and tb8_5.ts=tb7_5.ts and tb7_5.ts=tb6_5.ts and tb6_5.ts=tb5_5.ts and tb5_5.ts=tb4_5.ts and tb4_5.ts=tb3_5.ts and tb3_5.ts=tb2_5.ts and tb2_5.ts=tb1_5.ts and tb1_5.ts=tb0_5.ts; + +if $rows != 5 then + return -1 +endi + +if $data00 != @21-03-01 05:00:00.000@ then + return -1 +endi +if $data01 != 9905 then + return -1 +endi +if $data02 != 9905.000000000 then + return -1 +endi +if $data03 != 05 then + return -1 +endi +if $data04 != @21-03-01 05:00:00.000@ then + return -1 +endi +if $data05 != 9915 then + return -1 +endi +if $data06 != 9915.000000000 then + return -1 +endi +if $data07 != 15 then + return -1 +endi +if $data08 != @21-03-01 05:00:00.000@ then + return -1 +endi +if $data09 != 9925 then + return -1 +endi +if $data10 != @21-03-02 05:00:00.000@ then + return -1 +endi +if $data11 != 9905 then + return -1 +endi +if $data12 != 9905.000000000 then + return -1 +endi +if $data13 != 05 then + return -1 +endi +if $data14 != @21-03-02 05:00:00.000@ then + return -1 +endi +if $data15 != 9915 then + return -1 +endi +if $data16 != 9915.000000000 then + return -1 +endi +if $data17 != 15 then + return -1 +endi +if $data18 != @21-03-02 05:00:00.000@ then + return -1 +endi +if $data19 != 9925 then + return -1 +endi + + +sql_error select tb0_1.*, tb1_1.* from tb0_1, tb1_1 where tb0_1.f1=tb1_1.f1; +sql_error select tb0_1.*, tb1_1.* from tb0_1, tb1_1 where tb0_1.ts=tb1_1.ts and tb0_1.id1=tb1_1.id2; +sql_error select tb0_5.*, tb1_5.*,tb2_5.*,tb3_5.*,tb4_5.*,tb5_5.*, tb6_5.*,tb7_5.*,tb8_5.*,tb9_5.*,tba_5.* from tb0_5, tb1_5, tb2_5, tb3_5, tb4_5,tb5_5, tb6_5, tb7_5, tb8_5, tb9_5, tba_5 where tb9_5.ts=tb8_5.ts and tb8_5.ts=tb7_5.ts and tb7_5.ts=tb6_5.ts and tb6_5.ts=tb5_5.ts and tb5_5.ts=tb4_5.ts and tb4_5.ts=tb3_5.ts and tb3_5.ts=tb2_5.ts and tb2_5.ts=tb1_5.ts and tb1_5.ts=tb0_5.ts and tb0_5.ts=tba_5.ts; + +sql_error select * from st0, st1 where st0.ts=st1.ts; +sql_error select * from st0, st1 where st0.id1=st1.id1; +sql_error select * from st0, st1 where st0.f1=st1.f1 and st0.id1=st1.id1; +sql_error select * from st0, st1, st2, st3 where st0.id1=st1.id1 and st2.id1=st3.id1 and st0.ts=st1.ts and st1.ts=st2.ts and st2.ts=st3.ts; +sql_error select * from st0, st1, st2 where st0.id1=st1.id1; +sql_error select * from st0, st1 where st0.id1=st1.id1 and st0.id2=st1.id3; +sql_error select * from st0, st1 where st0.id1=st1.id1 or st0.ts=st1.ts; +sql_error select * from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 or st0.id2=st1.id2; +sql_error select * from st0, st1, st2 where st0.ts=st1.ts and st0.id1=st1.id1; +sql_error select * from st0, st1 where st0.id1=st1.ts and st0.ts=st1.id1; +sql_error select * from st0, st1 where st0.id1=st1.id2 and st0.ts=st1.ts; +sql_error select * from st0, st1 where st1.id4=st0.id4 and st1.ts=st0.ts; +sql_error select * from st0, st1 where st0.id1=st1.id2 and st1.ts=st0.ts; +sql_error select * from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 interval 10a; +sql_error select last(*) from st0, st1 where st0.ts=st1.ts and st0.id1=st1.id1 group by f1; +sql_error select st0.*,st1.*,st2.*,st3.*,st4.*,st5.*,st6.*,st7.*,st8.*,st9.* from st0,st1,st2,st3,st4,st5,st6,st7,st8,st9 where st0.ts=st2.ts and st0.ts=st4.ts and st0.ts=st6.ts and st0.ts=st8.ts and st1.ts=st3.ts and st3.ts=st5.ts and st5.ts=st7.ts and st7.ts=st9.ts and st0.id1=st2.id1 and st0.id1=st4.id1 and st0.id1=st6.id1 and st0.id1=st8.id1 and st1.id1=st3.id1 and st3.id1=st5.id1 and st5.id1=st7.id1 and st7.id1=st9.id1; +sql_error select st0.*,st1.*,st2.*,st3.*,st4.*,st5.*,st6.*,st7.*,st8.*,st9.* from st0,st1,st2,st3,st4,st5,st6,st7,st8,st9,sta where st0.ts=st2.ts and st0.ts=st4.ts and st0.ts=st6.ts and st0.ts=st8.ts and st1.ts=st3.ts and st3.ts=st5.ts and st5.ts=st7.ts and st7.ts=st9.ts and st0.ts=st1.ts and st0.id1=st2.id1 and st0.id1=st4.id1 and st0.id1=st6.id1 and st0.id1=st8.id1 and st1.id1=st3.id1 and st3.id1=st5.id1 and st5.id1=st7.id1 and st7.id1=st9.id1 and st0.id1=st1.id1 and st0.id1=sta.id1 and st0.ts=sta.ts; + + + + + + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index 9d7cdfe3d7598a77ade1861fe6a96513f6c828f1..2322496a94467d44e3fdc5eabe29dded23e7dbde 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/general/parser/last_groupby.sim b/tests/script/general/parser/last_groupby.sim new file mode 100644 index 0000000000000000000000000000000000000000..f993324cd1ccb6e1d74f71b3a0d9b47cf3452b48 --- /dev/null +++ b/tests/script/general/parser/last_groupby.sim @@ -0,0 +1,99 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$db = testdb + +sql create database $db +sql use $db + +sql create stable st2 (ts timestamp, f1 int, f2 float, f3 double, f4 bigint, f5 smallint, f6 tinyint, f7 bool, f8 binary(10), f9 nchar(10)) tags (id1 int, id2 float, id3 nchar(10), id4 double, id5 smallint, id6 bigint, id7 binary(10)) + +sql create table tb1 using st2 tags (1,1.0,"1",1.0,1,1,"1"); + +sql insert into tb1 values (now-200s,1,1.0,1.0,1,1,1,true,"1","1") +sql insert into tb1 values (now-100s,2,2.0,2.0,2,2,2,true,"2","2") +sql insert into tb1 values (now,3,3.0,3.0,3,3,3,true,"3","3") +sql insert into tb1 values (now+100s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+200s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+300s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+400s,4,4.0,4.0,4,4,4,true,"4","4") +sql insert into tb1 values (now+500s,4,4.0,4.0,4,4,4,true,"4","4") + +sql select f1,last(*) from st2 group by f1; + +if $rows != 4 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data02 != 1 then + print $data02 + return -1 +endi +if $data03 != 1.00000 then + return -1 +endi +if $data04 != 1.000000000 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data06 != 1 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != 1 then + return -1 +endi +if $data09 != 1 then + return -1 +endi + +sql select f1,last(f1,st2.*) from st2 group by f1; +if $rows != 4 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1.00000 then + return -1 +endi +if $data05 != 1.000000000 then + return -1 +endi +if $data06 != 1 then + return -1 +endi +if $data07 != 1 then + return -1 +endi +if $data08 != 1 then + return -1 +endi +if $data09 != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/lastrow.sim b/tests/script/general/parser/lastrow.sim index f997dc504fdecf997d316059257ec73d4047aaf0..2b8f294d5d058f4b7cc8c45380862180e35a5899 100644 --- a/tests/script/general/parser/lastrow.sim +++ b/tests/script/general/parser/lastrow.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -62,7 +62,7 @@ run general/parser/lastrow_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index a87b3cc646871e442cd3540fa163f1d5492c1456..3fd88cfc1bb4e651ae16801c249aa88f0a3797ba 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -16,6 +16,9 @@ $stb = $stbPrefix . $i sql use $db +print ========>TD-3231 last_row with group by column error +sql_error select last_row(c1) from $stb group by c1; + ##### select lastrow from STable with two vnodes, timestamp decreases from tables in vnode0 to tables in vnode1 sql select last_row(*) from $stb if $rows != 1 then @@ -224,4 +227,4 @@ sql create table tu(ts timestamp, k int) sql select last_row(*) from tu if $row != 0 then return -1 -endi \ No newline at end of file +endi diff --git a/tests/script/general/parser/limit.sim b/tests/script/general/parser/limit.sim index 682b449ca3c97a7ecff0d047b64361629dcfb858..23b85095c54c9bb439c0766e379b8950778f8d90 100644 --- a/tests/script/general/parser/limit.sim +++ b/tests/script/general/parser/limit.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -66,7 +66,7 @@ run general/parser/limit_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/limit1.sim b/tests/script/general/parser/limit1.sim index c047dc28443bb06ca4efbe6c6bb1e366ec4b8e3b..e37bea92207d5a3da34033bbe135570344751375 100644 --- a/tests/script/general/parser/limit1.sim +++ b/tests/script/general/parser/limit1.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -61,7 +61,7 @@ run general/parser/limit1_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/limit1_stb.sim b/tests/script/general/parser/limit1_stb.sim index a0e3a45d2fc28002e2e64b195191b72659849a53..513e2fac026c0d1d617ba4126d22c01f82527ca0 100644 --- a/tests/script/general/parser/limit1_stb.sim +++ b/tests/script/general/parser/limit1_stb.sim @@ -538,6 +538,7 @@ $offset = $offset + 1 sql select max(c1), min(c2), avg(c3), count(c4), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and t1 > 1 and t1 < 8 interval(5m) limit $offset offset $offset $val = $rowNum - $offset if $rows != $val then + print expect $val, actual:$rows return -1 endi if $data00 != @18-10-22 02:30:00.000@ then diff --git a/tests/script/general/parser/limit1_tblocks100.sim b/tests/script/general/parser/limit1_tblocks100.sim index 039a171ad703998499e9492e3633741cee3fa6e5..4546ffdb7910a7dd999a119743fa36682fe0eade 100644 --- a/tests/script/general/parser/limit1_tblocks100.sim +++ b/tests/script/general/parser/limit1_tblocks100.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -61,7 +61,7 @@ run general/parser/limit1_stb.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/limit2.sim b/tests/script/general/parser/limit2.sim index f9b46f5c3e187179ac25eb666696ee3caeb7aa3d..336f9234c82ab51b1ff478e21f99e058074a2818 100644 --- a/tests/script/general/parser/limit2.sim +++ b/tests/script/general/parser/limit2.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c rowsInFileBlock -v 255 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -69,7 +69,7 @@ print ====== tables created print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/limit2_query.sim b/tests/script/general/parser/limit2_query.sim index 1e8077d26e03a9fb50425a74bbbe0ca027fb9c66..9fe287960d22642bbea0139246d3f90537fef628 100644 --- a/tests/script/general/parser/limit2_query.sim +++ b/tests/script/general/parser/limit2_query.sim @@ -143,6 +143,97 @@ if $data11 != -1 then return -1 endi +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 8200 +if $rows != 8200 then + return -1 +endi + +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 10 offset 8190; +if $rows != 10 then + return -1 +endi + +if $data00 != @18-10-15 19:30:00.000@ then + return -1 +endi + +if $data01 != 5 then + return -1 +endi + +if $data10 != @18-10-15 19:35:00.000@ then + return -1 +endi + +if $data11 != -1000 then + return -1 +endi + +if $data20 != @18-10-15 19:40:00.000@ then + return -1 +endi + +if $data21 != 6 then + return -1 +endi + +if $data30 != @18-10-15 19:45:00.000@ then + return -1 +endi + +if $data31 != -1000 then + return -1 +endi + +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 10 offset 10001; +if $rows != 10 then + return -1 +endi + +if $data00 != @18-10-22 02:25:00.000@ then + return -1 +endi + +if $data01 != -1000 then + return -1 +endi + +if $data10 != @18-10-22 02:30:00.000@ then + return -1 +endi + +if $data11 != 1 then + return -1 +endi + +if $data20 != @18-10-22 02:35:00.000@ then + return -1 +endi + +if $data21 != -1000 then + return -1 +endi + +if $data30 != @18-10-22 02:40:00.000@ then + return -1 +endi + +if $data31 != 2 then + return -1 +endi + +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 10000 offset 10001; +print ====> needs to validate the last row result +if $rows != 9998 then + return -1 +endi + + +sql select max(c1) from lm2_tb0 where ts >= 1537146000000 and ts <= 1543145400000 interval(5m) fill(value, -1000, -2) limit 100 offset 20001; +if $rows != 0 then + return -1 +endi + # tb + interval + fill(linear) + limit offset $limit = $rowNum $offset = $limit / 2 diff --git a/tests/script/general/parser/limit2_tblocks100.sim b/tests/script/general/parser/limit2_tblocks100.sim index 19cea74e708beadcd1c9185b8b7d911f3e4f872b..11f7a15eb06a3de4bc5ab2eda6acade17abe5000 100644 --- a/tests/script/general/parser/limit2_tblocks100.sim +++ b/tests/script/general/parser/limit2_tblocks100.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c rowsInFileBlock -v 255 system sh/exec.sh -n dnode1 -s start sleep 100 diff --git a/tests/script/general/parser/mixed_blocks.sim b/tests/script/general/parser/mixed_blocks.sim index 772dc1db59c0d8bdcaaf63e11495c9a7d4572929..c20cf9a915e49d014e29ce4bb817802f22346f9e 100644 --- a/tests/script/general/parser/mixed_blocks.sim +++ b/tests/script/general/parser/mixed_blocks.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start @@ -59,7 +59,7 @@ sql show databases print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 2000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed @@ -154,7 +154,7 @@ sql insert into t2 values('2020-1-1 1:5:1', 99); print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql select ts from m1 where ts='2020-1-1 1:5:1' diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index 3dcfca7503cdf82b8896a715e1393e614a498c34..786cee651b793b23ec8519be400554567af49852 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 diff --git a/tests/script/general/parser/null_char.sim b/tests/script/general/parser/null_char.sim index 4e7c8ab3548a54a465801c602c0d6572b347b284..cb65290d2548c4ca6e7377d18b33d5c1768e818e 100644 --- a/tests/script/general/parser/null_char.sim +++ b/tests/script/general/parser/null_char.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 diff --git a/tests/script/general/parser/projection_limit_offset.sim b/tests/script/general/parser/projection_limit_offset.sim index df5be140f6dc77a9ed961c1a4555b200e08ae5ab..ffbcb28ffd9b4e15f707509dc5cc808ef3f8ce4a 100644 --- a/tests/script/general/parser/projection_limit_offset.sim +++ b/tests/script/general/parser/projection_limit_offset.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -334,6 +334,9 @@ sql insert into tm0 values(10000, 1) (20000, 2)(30000, 3) (40000, NULL) (50000, #=============================tbase-1205 sql select count(*) from tm1 where ts= now -1d interval(1h) fill(NULL); +if $rows != 0 then + return -1 +endi print ===================>TD-1834 sql select * from tm0 where ts>11000 and ts< 20000 order by ts asc @@ -409,7 +412,7 @@ sql_error select k, sum(k)+1 from tm0; print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/selectResNum.sim b/tests/script/general/parser/selectResNum.sim index 071dd87bc975532ea0032cb54fff6aca1408ff8e..dfd204e15240c93c1f9bcde32b8eb65c0918604a 100644 --- a/tests/script/general/parser/selectResNum.sim +++ b/tests/script/general/parser/selectResNum.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -118,7 +118,7 @@ endw print ====== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ====== server restart completed sleep 100 diff --git a/tests/script/general/parser/select_across_vnodes.sim b/tests/script/general/parser/select_across_vnodes.sim index 6c473a35d14cccf5cb24792547649500a9935563..9bf61ee61d16a5220b6ae501b8c9d345f9131c1e 100644 --- a/tests/script/general/parser/select_across_vnodes.sim +++ b/tests/script/general/parser/select_across_vnodes.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 system sh/exec.sh -n dnode1 -s start sleep 100 diff --git a/tests/script/general/parser/select_distinct_tag.sim b/tests/script/general/parser/select_distinct_tag.sim new file mode 100644 index 0000000000000000000000000000000000000000..d8e92d4bc5ed3e3a1def0b33faf23ec66047227d --- /dev/null +++ b/tests/script/general/parser/select_distinct_tag.sim @@ -0,0 +1,61 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5 +system sh/exec.sh -n dnode1 -s start +sleep 100 +sql connect + +$dbPrefix = sav_db +$tbPrefix = sav_tb +$stbPrefix = sav_stb +$tbNum = 20 +$rowNum = 10 +$totalNum = $tbNum * $rowNum +$ts0 = 1537146000000 +$delta = 600000 +print ========== alter.sim +$i = 0 +$db = $dbPrefix +$stb = $stbPrefix + +sql drop database if exists $db +sql create database $db +sql use $db +print ====== create tables +sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int, t2 int) + +$i = 0 +$ts = $ts0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $stb tags( $i , 0 ) + $i = $i + 1 +endw + +print ====== table created + +#### select distinct tag +sql select distinct t1 from $stb +if $rows != $tbNum then + return -1 +endi + +#### select distinct tag +sql select distinct t2 from $stb +if $rows != 1 then + print $rows + return -1 +endi + +#### unsupport sql +sql_error select distinct t1, t2 from &stb + +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/select_from_cache_disk.sim b/tests/script/general/parser/select_from_cache_disk.sim index 5feae919059b72a877a789158055370887343659..7f8af52c6bd395416f982554083a81ee1939f059 100644 --- a/tests/script/general/parser/select_from_cache_disk.sim +++ b/tests/script/general/parser/select_from_cache_disk.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -35,7 +35,7 @@ sql insert into $tb values ('2018-09-17 09:00:00.030', 3) print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim index 5428f98593ae473f9df1038ba4d68a1a2ac38386..38a514a51b0fce41e003a1040bf264eeca3bf29a 100644 --- a/tests/script/general/parser/select_with_tags.sim +++ b/tests/script/general/parser/select_with_tags.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 100 diff --git a/tests/script/general/parser/set_tag_vals.sim b/tests/script/general/parser/set_tag_vals.sim index 0b8ffd946f2bb052743faf4a732a06e4a68c5aec..74184f94d47b07684a233ef692769094141a3f56 100644 --- a/tests/script/general/parser/set_tag_vals.sim +++ b/tests/script/general/parser/set_tag_vals.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -61,7 +61,7 @@ while $i < $tbNum endw print ====== tables created -sleep 2000 +sleep 500 sql show tables if $rows != $tbNum then diff --git a/tests/script/general/parser/single_row_in_tb.sim b/tests/script/general/parser/single_row_in_tb.sim index 6f1535c390ac18614e9da07d74e390ab39099303..5de2a51f0f81d9286184c11e97d682a2b82ebdcd 100644 --- a/tests/script/general/parser/single_row_in_tb.sim +++ b/tests/script/general/parser/single_row_in_tb.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -32,7 +32,7 @@ run general/parser/single_row_in_tb_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/sliding.sim b/tests/script/general/parser/sliding.sim index be33c905a1786b8c56d21aa973e6eb1fb8698ad6..b2695ff95f27c5fe6ba27c9b977c96b48a5e5801 100644 --- a/tests/script/general/parser/sliding.sim +++ b/tests/script/general/parser/sliding.sim @@ -469,11 +469,12 @@ if $data25 != 33 then endi sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts desc limit 1000; -if $row != 1 then +if $row != 3 then return -1 endi -if $data00 != @99-12-31 23:59:40.000@ then +if $data00 != @00-01-01 00:00:00.000@ then + print expect 00-01-01 00:00:00.000, actual: $data00 return -1 endi @@ -489,7 +490,28 @@ if $data03 != 33 then return -1 endi -print check boundary check crash at client side +if $data10 != @99-12-31 23:59:50.000@ then + return -1 +endi + +if $data11 != 33 then + return -1 +endi +if $data12 != 9.521904571 then + return -1 +endi + +if $data20 != @99-12-31 23:59:40.000@ then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data22 != 9.521904571 then + return -1 +endi + +print ====================>check boundary check crash at client side sql select count(*) from sliding_mt0 where ts>now and ts < now-1h; print ========================query on super table diff --git a/tests/script/general/parser/slimit.sim b/tests/script/general/parser/slimit.sim index 9aaf4a35caf32c67e2e1c04f5182074465ba11ad..0af31f982604a3b6c6e0901cd94795fc503edd4b 100644 --- a/tests/script/general/parser/slimit.sim +++ b/tests/script/general/parser/slimit.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -97,7 +97,7 @@ run general/parser/slimit_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/slimit1.sim b/tests/script/general/parser/slimit1.sim index 2c8fa28d326aa97cb3e3e740961c381eda34e09d..2dede439ec38c862afcce5e1658fa02824282087 100644 --- a/tests/script/general/parser/slimit1.sim +++ b/tests/script/general/parser/slimit1.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -56,7 +56,7 @@ run general/parser/slimit1_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/slimit_alter_tags.sim b/tests/script/general/parser/slimit_alter_tags.sim index 3fe40dbe2e3f46a06da59753cfd8b40339771735..53af0f3e8470b0708f2193b09ed1797c8abc9a52 100644 --- a/tests/script/general/parser/slimit_alter_tags.sim +++ b/tests/script/general/parser/slimit_alter_tags.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 system sh/exec.sh -n dnode1 -s start sleep 100 @@ -93,7 +93,7 @@ if $data02 != tb0 then return -1 endi -sleep 2000 +sleep 500 sql reset query cache sql select count(*), first(ts) from stb group by tg_added order by tg_added asc slimit 5 soffset 3 if $rows != 5 then @@ -171,7 +171,7 @@ endi print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect diff --git a/tests/script/general/parser/stableOp.sim b/tests/script/general/parser/stableOp.sim new file mode 100644 index 0000000000000000000000000000000000000000..8647657e7bee1ea73dcdbdc6f346e2279dd58cd5 --- /dev/null +++ b/tests/script/general/parser/stableOp.sim @@ -0,0 +1,95 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect +print ======================== dnode1 start + +$dbPrefix = fi_in_db +$tbPrefix = fi_in_tb +$stbPrefix = fi_in_stb +$mtPrefix = fi_in_mt +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print create_tb test +print =============== set up +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i +$tb = $tbPrefix . $i +$stb = $stbPrefix . $i + +sql create database $db +sql use $db + +# case1: create stable test +print =========== stableOp.sim case1: create/alter/drop stable test +sql CREATE STABLE $stb (TS TIMESTAMP, COL1 INT) TAGS (ID INT); +sql show stables + +if $rows != 1 then + return -1 +endi +print data00 = $data00 +if $data00 != $stb then + return -1 +endi + +sql_error CREATE STABLE $tb using $stb tags (1); + +sql create table $tb using $stb tags (2); +sql show tables + +if $rows != 1 then + return -1 +endi + +sql alter stable $stb add column COL2 DOUBLE; + +sql insert into $tb values (now, 1, 2.0); + +sql select * from $tb ; + +if $rows != 1 then + return -1 +endi + +sql alter stable $stb drop column COL2; + +sql_error insert into $tb values (now, 1, 2.0); + +sql alter stable $stb add tag tag2 int; + +sql alter stable $stb change tag tag2 tag3; + +sql_error drop stable $tb + +sql drop table $tb ; + +sql show tables + +if $rows != 0 then + return -1 +endi + +sql DROP STABLE $stb +sql show stables + +if $rows != 0 then + return -1 +endi + +print create/alter/drop stable test passed + +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/tags_dynamically_specifiy.sim b/tests/script/general/parser/tags_dynamically_specifiy.sim index 87b278da095463e2c51bb4c0c8bec8b22d2e873c..f6b3dabf153e502b598d1bea8545db3232ceef2d 100644 --- a/tests/script/general/parser/tags_dynamically_specifiy.sim +++ b/tests/script/general/parser/tags_dynamically_specifiy.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect diff --git a/tests/script/general/parser/tags_filter.sim b/tests/script/general/parser/tags_filter.sim index f0ac3bdcbafcc06c7040215fa4d9e54948ac0758..3d3e79b6f52928af9d3333810f57300e52e0ebb0 100644 --- a/tests/script/general/parser/tags_filter.sim +++ b/tests/script/general/parser/tags_filter.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect diff --git a/tests/script/general/parser/tbnameIn.sim b/tests/script/general/parser/tbnameIn.sim index e1d200e71688e8e633c97e6ada59848cd2b0ec44..003a86f90b2f36602b4e999aee2974ef259d3670 100644 --- a/tests/script/general/parser/tbnameIn.sim +++ b/tests/script/general/parser/tbnameIn.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 100 sql connect @@ -67,7 +67,7 @@ run general/parser/tbnameIn_query.sim print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 2d77cb15d248ee805f0b214960d18e06e3ac4d7e..f05474d15804fba661f650d69632c454415eba64 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,107 +1,58 @@ -#run general/parser/alter.sim -#sleep 100 -#run general/parser/alter1.sim -#sleep 100 -#run general/parser/alter_stable.sim -#sleep 100 -#run general/parser/auto_create_tb.sim -#sleep 100 -#run general/parser/auto_create_tb_drop_tb.sim -#sleep 100 -#run general/parser/col_arithmetic_operation.sim -#sleep 100 -#run general/parser/columnValue.sim -#sleep 100 -#run general/parser/commit.sim -#sleep 100 -#run general/parser/create_db.sim -#sleep 100 -#run general/parser/create_mt.sim -#sleep 100 -#run general/parser/create_tb.sim -#sleep 100 -#run general/parser/dbtbnameValidate.sim -#sleep 100 -#run general/parser/fill.sim -#sleep 100 -#run general/parser/fill_stb.sim -#sleep 100 -##run general/parser/fill_us.sim # -#sleep 100 -#run general/parser/first_last.sim -#sleep 100 -#run general/parser/import_commit1.sim -#sleep 100 -#run general/parser/import_commit2.sim -#sleep 100 -#run general/parser/import_commit3.sim -#sleep 100 -##run general/parser/import_file.sim -#sleep 100 -#run general/parser/insert_tb.sim -#sleep 100 -#run general/parser/tags_dynamically_specifiy.sim -#sleep 100 -#run general/parser/interp.sim -#sleep 100 -#run general/parser/lastrow.sim -#sleep 100 -#run general/parser/limit.sim -#sleep 100 -#run general/parser/limit1.sim -#sleep 100 -#run general/parser/limit1_tblocks100.sim -#sleep 100 -#run general/parser/limit2.sim -#sleep 100 -#run general/parser/mixed_blocks.sim -#sleep 100 -#run general/parser/nchar.sim -#sleep 100 -#run general/parser/null_char.sim -sleep 100 +run general/parser/alter.sim +run general/parser/alter1.sim +run general/parser/alter_stable.sim +run general/parser/auto_create_tb.sim +run general/parser/auto_create_tb_drop_tb.sim +run general/parser/col_arithmetic_operation.sim +run general/parser/columnValue.sim +run general/parser/commit.sim +run general/parser/create_db.sim +run general/parser/create_mt.sim +run general/parser/create_tb.sim +run general/parser/dbtbnameValidate.sim +run general/parser/fill.sim +run general/parser/fill_stb.sim +#run general/parser/fill_us.sim # +run general/parser/first_last.sim +run general/parser/import_commit1.sim +run general/parser/import_commit2.sim +run general/parser/import_commit3.sim +#run general/parser/import_file.sim +run general/parser/insert_tb.sim +run general/parser/tags_dynamically_specifiy.sim +run general/parser/interp.sim +run general/parser/lastrow.sim +run general/parser/limit.sim +run general/parser/limit1.sim +run general/parser/limit1_tblocks100.sim +run general/parser/limit2.sim +run general/parser/mixed_blocks.sim +run general/parser/nchar.sim +run general/parser/null_char.sim run general/parser/selectResNum.sim -sleep 100 run general/parser/select_across_vnodes.sim -sleep 100 run general/parser/select_from_cache_disk.sim -sleep 100 run general/parser/set_tag_vals.sim -sleep 100 run general/parser/single_row_in_tb.sim -sleep 100 run general/parser/slimit.sim -sleep 100 run general/parser/slimit1.sim -sleep 100 run general/parser/slimit_alter_tags.sim -sleep 100 run general/parser/tbnameIn.sim -sleep 100 run general/parser/slimit_alter_tags.sim # persistent failed -sleep 100 run general/parser/join.sim -sleep 100 run general/parser/join_multivnode.sim -sleep 100 +run general/parser/join_manyblocks.sim run general/parser/projection_limit_offset.sim -sleep 100 run general/parser/select_with_tags.sim -sleep 100 run general/parser/groupby.sim -sleep 100 run general/parser/tags_filter.sim -sleep 100 run general/parser/topbot.sim -sleep 100 run general/parser/union.sim -sleep 100 run general/parser/constCol.sim -sleep 100 run general/parser/where.sim -sleep 100 run general/parser/timestamp.sim -sleep 100 run general/parser/sliding.sim -sleep 100 run general/parser/function.sim +run general/parser/stableOp.sim +run general/parser/slimit_alter_tags.sim + diff --git a/tests/script/general/parser/timestamp.sim b/tests/script/general/parser/timestamp.sim index 7d7362bcb5200f96f6b216430d5f712812071e66..0a87bce51dedecf0c39179ffc7b1aa864b5e7823 100644 --- a/tests/script/general/parser/timestamp.sim +++ b/tests/script/general/parser/timestamp.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index 6188f42ff51e87b19ced47ab2ae3cae792bb2ace..5c61acb2ba246f1f0102ed01b90e0b164ebfa8dc 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 200 system sh/exec.sh -n dnode1 -s start @@ -73,6 +73,60 @@ if $row != 100 then return -1 endi +sql select bottom(c3, 5) from tb_tb1 interval(1y); +if $rows != 5 then + return -1 +endi + +if $data01 != 0.00000 then + print expect 0.00000, actual:$data01 + return -1 +endi + +if $data11 != 0.00000 then + return -1 +endi + +if $data21 != 0.00000 then + return -1 +endi + +if $data31 != 0.00000 then + return -1 +endi + +sql select top(c4, 5) from tb_tb1 interval(1y); +if $rows != 5 then + return -1 +endi + +if $data01 != 9.000000000 then + print expect 9.000000000, acutal:$data01 + return -1 +endi + +if $data11 != 9.000000000 then + return -1 +endi + +if $data21 != 9.000000000 then + return -1 +endi + +if $data31 != 9.000000000 then + return -1 +endi + +sql select top(c3, 5) from tb_tb1 interval(40h) +if $rows != 25 then + return -1 +endi + +if $data01 != 9.00000 then + print expect 9.00000, actual:$data01 + return -1 +endi + sql select last(*) from tb_tb9 if $row != 1 then return -1 @@ -128,7 +182,7 @@ sql insert into test values(29999, 1)(70000, 2)(80000, 3) print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 +sleep 500 system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect @@ -262,4 +316,65 @@ if $data13 != @20-02-02 01:01:01.000@ then return -1 endi +print ===============================>td-3361 +sql create table ttm1(ts timestamp, k int) tags(a nchar(12)); +sql create table ttm1_t1 using ttm1 tags('abcdef') +sql insert into ttm1_t1 values(now, 1) +sql select * from ttm1 where a=123456789012 +if $row != 0 then + return -1 +endi + +print ===============================>td-3621 +sql create table ttm2(ts timestamp, k bool); +sql insert into ttm2 values('2021-1-1 1:1:1', true) +sql insert into ttm2 values('2021-1-1 1:1:2', NULL) +sql insert into ttm2 values('2021-1-1 1:1:3', false) +sql select * from ttm2 where k is not null +if $row != 2 then + return -1 +endi + +if $data00 != @21-01-01 01:01:01.000@ then + print expect 21-01-01 01:01:01.000, actual $data00 + return -1 +endi + +sql select * from ttm2 where k is null +if $row != 1 then + return -1 +endi + +if $data00 != @21-01-01 01:01:02.000@ then + return -1 +endi + +sql select * from ttm2 where k=true +if $row != 1 then + return -1 +endi + +if $data00 != @21-01-01 01:01:01.000@ then + return -1 +endi + +sql select * from ttm2 where k=false +if $row != 1 then + return -1 +endi + +if $data00 != @21-01-01 01:01:03.000@ then + return -1 +endi + +sql select * from ttm2 where k<>false +if $row != 1 then + return -1 +endi + +sql_error select * from ttm2 where k=null +sql_error select * from ttm2 where k<>null +sql_error select * from ttm2 where k like null +sql_error select * from ttm2 where ktd-3318 +sql create table tu(ts timestamp, k int, b binary(12)) +sql insert into tu values(now, 1, 'abc') +sql select stddev(k) from tu where b <>'abc' interval(1s) +if $rows != 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stable/disk.sim b/tests/script/general/stable/disk.sim index a67ef6d79097706f90c4a2eb229d5329d785e24c..1faae78e898107af6d3a1140413ffe16b0680e26 100644 --- a/tests/script/general/stable/disk.sim +++ b/tests/script/general/stable/disk.sim @@ -2,12 +2,12 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -58,7 +58,7 @@ endi sleep 1000 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start sleep 6000 diff --git a/tests/script/general/stable/dnode3.sim b/tests/script/general/stable/dnode3.sim index 2859f644bb883483385fdf2421b0996a4416b9e6..872cfb8d2e5022872e30eaabd7db67a4546a0204 100644 --- a/tests/script/general/stable/dnode3.sim +++ b/tests/script/general/stable/dnode3.sim @@ -4,10 +4,10 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 diff --git a/tests/script/general/stable/metrics.sim b/tests/script/general/stable/metrics.sim index b94c76429dcae6dbd39d80beab03b4a5bc8b401f..a3dca3f1a5a5f2234520e987ab356f6ed68f0afc 100644 --- a/tests/script/general/stable/metrics.sim +++ b/tests/script/general/stable/metrics.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_me_db diff --git a/tests/script/general/stable/refcount.sim b/tests/script/general/stable/refcount.sim index 5fe95dde9fdf8bd8820b067f0fc30d01b5e89f80..6629dc1177c6021fce45f135f153475f6cb858a3 100644 --- a/tests/script/general/stable/refcount.sim +++ b/tests/script/general/stable/refcount.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 diff --git a/tests/script/general/stable/show.sim b/tests/script/general/stable/show.sim index 66755edea73f30a3d3ecfc2dc090f0588dc21947..5fe05b41eb8fb6f0080eeea35cc63506faf55e5c 100644 --- a/tests/script/general/stable/show.sim +++ b/tests/script/general/stable/show.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== create stable diff --git a/tests/script/general/stable/values.sim b/tests/script/general/stable/values.sim index 51488aabef8918b957ca63fd29823b14f772b675..fb2c908da269872ddd85489d9d98510e1bb58749 100644 --- a/tests/script/general/stable/values.sim +++ b/tests/script/general/stable/values.sim @@ -2,11 +2,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stable/vnode3.sim b/tests/script/general/stable/vnode3.sim index 3409aef9f88bf652979d5b16c023417c4820168f..61948b506302f803aee2f73eb5e528e14f1174db 100644 --- a/tests/script/general/stable/vnode3.sim +++ b/tests/script/general/stable/vnode3.sim @@ -1,12 +1,12 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/agg_stream.sim b/tests/script/general/stream/agg_stream.sim index 814438ac74531c565327bca6bb9615928bf289e1..65657fc33be88152040ce68c46d559e8b2c119a5 100644 --- a/tests/script/general/stream/agg_stream.sim +++ b/tests/script/general/stream/agg_stream.sim @@ -13,7 +13,7 @@ system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step2 diff --git a/tests/script/general/stream/column_stream.sim b/tests/script/general/stream/column_stream.sim index cb94904ff27903acb57b2202c790d25a9b1877ba..c43ca1fd5ac7cfc5e477c970b313414469e9a29c 100644 --- a/tests/script/general/stream/column_stream.sim +++ b/tests/script/general/stream/column_stream.sim @@ -9,7 +9,7 @@ system sh/cfg.sh -n dnode1 -c monitor -v 1 system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -121,7 +121,7 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start print sleep 22 seconds sleep 22000 diff --git a/tests/script/general/stream/metrics_del.sim b/tests/script/general/stream/metrics_del.sim index e21fa5999aa1b6ce0c063af54172f222aeb983e1..321658cd8da686642e35936e0d622079aa9d7f1f 100644 --- a/tests/script/general/stream/metrics_del.sim +++ b/tests/script/general/stream/metrics_del.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/metrics_replica1_vnoden.sim b/tests/script/general/stream/metrics_replica1_vnoden.sim index dcbc8ae7de4962c8970cdffec5829c74cdfb7351..4629063c4460c10dccc844b6b39e512012638379 100644 --- a/tests/script/general/stream/metrics_replica1_vnoden.sim +++ b/tests/script/general/stream/metrics_replica1_vnoden.sim @@ -2,12 +2,12 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/restart_stream.sim b/tests/script/general/stream/restart_stream.sim index b5a2038d9b207217f9a41250939b9853e2e822d1..c8be10103d16a856fb0c083546cfaf8b939a1b98 100644 --- a/tests/script/general/stream/restart_stream.sim +++ b/tests/script/general/stream/restart_stream.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -96,12 +96,13 @@ endi print =============== step4 system sh/exec.sh -n dnode1 -s stop system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start print =============== step5 -print ==> sleep 8 seconds to renew cache -sleep 8000 +print ==> renew cache +sql reset query cache +sleep 1000 print =============== step6 diff --git a/tests/script/general/stream/stream_3.sim b/tests/script/general/stream/stream_3.sim index 88105a77d667e9cad499ec55cad7f0d7f8fc0c3f..31490dc5ac44b93f0daf02cd0ba3b4d77a3be20b 100644 --- a/tests/script/general/stream/stream_3.sim +++ b/tests/script/general/stream/stream_3.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -108,7 +108,7 @@ print =============== step7 system sh/exec.sh -n dnode1 -s stop system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sleep 4000 diff --git a/tests/script/general/stream/stream_restart.sim b/tests/script/general/stream/stream_restart.sim index 480b23055e90f59bd597d3319a5fe139eba83bab..4bf6760703ef744572a0b172cb860141f8768ff5 100644 --- a/tests/script/general/stream/stream_restart.sim +++ b/tests/script/general/stream/stream_restart.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/table_del.sim b/tests/script/general/stream/table_del.sim index ce4065a1a85ba2c4f8f79b77d1d82c5118091e9c..3cbce538d5a87553289b834bd15dea1f43da35fb 100644 --- a/tests/script/general/stream/table_del.sim +++ b/tests/script/general/stream/table_del.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/stream/table_replica1_vnoden.sim b/tests/script/general/stream/table_replica1_vnoden.sim index 13a4a56fb300e877c01bbe52e15aa8221d246cb5..be67a31b4e6782e81289b03fb267db501f341520 100644 --- a/tests/script/general/stream/table_replica1_vnoden.sim +++ b/tests/script/general/stream/table_replica1_vnoden.sim @@ -2,12 +2,12 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 1000 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 3 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/table/autocreate.sim b/tests/script/general/table/autocreate.sim index 8469a6484ac5aeb7e656ad63164ebfb0546af646..404c714ab4cfa85593d191fcd528f7b902e4cea9 100644 --- a/tests/script/general/table/autocreate.sim +++ b/tests/script/general/table/autocreate.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database diff --git a/tests/script/general/table/basic1.sim b/tests/script/general/table/basic1.sim index e8d0cd7bd8bd0b6f3dd7193b12a33c70bb46f95b..c26c3c33e4102723805925cccb61c63d01c866b8 100644 --- a/tests/script/general/table/basic1.sim +++ b/tests/script/general/table/basic1.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database diff --git a/tests/script/general/table/basic2.sim b/tests/script/general/table/basic2.sim index d1678e8abd984b3aaa5061feb9430550d8909feb..4286f9ee4a5fce5ae98956d8f2e4d143000cbdbf 100644 --- a/tests/script/general/table/basic2.sim +++ b/tests/script/general/table/basic2.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== one table diff --git a/tests/script/general/table/basic3.sim b/tests/script/general/table/basic3.sim index ded00e153a2be88ed8c9193d4c9051dacde1fc31..41c276ae984c73672c439f39ba7c54f94f375ef4 100644 --- a/tests/script/general/table/basic3.sim +++ b/tests/script/general/table/basic3.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== create database diff --git a/tests/script/general/table/bigint.sim b/tests/script/general/table/bigint.sim index dd61d25ef9f2c300a33ef8a30bcc17a1ab5d867d..d75f406d77688374d75fd317508dd114f965f73a 100644 --- a/tests/script/general/table/bigint.sim +++ b/tests/script/general/table/bigint.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/binary.sim b/tests/script/general/table/binary.sim index cdbfcd4cbfe15438db884cd69b149c6e3519b6e6..47915530efc618f959c4f4ce17ea130aaa6ddf30 100644 --- a/tests/script/general/table/binary.sim +++ b/tests/script/general/table/binary.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/bool.sim b/tests/script/general/table/bool.sim index fbb6fe823cf30fc18302855aca9e847373cd1f6f..e49637448f8f6afea890e58da47b81db20b35768 100644 --- a/tests/script/general/table/bool.sim +++ b/tests/script/general/table/bool.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/column2.sim b/tests/script/general/table/column2.sim index 9251e31daa633b45dae6ea7c6240d6c7a731f405..441766f2d4e6a2ff565b56f10b623e35b7c4c2ad 100644 --- a/tests/script/general/table/column2.sim +++ b/tests/script/general/table/column2.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 diff --git a/tests/script/general/table/column_name.sim b/tests/script/general/table/column_name.sim index 0b09f65129d2204c4088e5aff55c08384988a2b5..47fcfab5a8979bfb91290429a0a6490368371685 100644 --- a/tests/script/general/table/column_name.sim +++ b/tests/script/general/table/column_name.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/column_num.sim b/tests/script/general/table/column_num.sim index d1091528b249cc35f31ab623b29d4dc3148c9836..a18173bc8f866fc984a1b10879990d69eaf0a311 100644 --- a/tests/script/general/table/column_num.sim +++ b/tests/script/general/table/column_num.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/column_value.sim b/tests/script/general/table/column_value.sim index 117c288b3602d3ca61fe83591c6943b79903e779..1edf8c2992c4e5fb3dacdbac627a8687e1fc549f 100644 --- a/tests/script/general/table/column_value.sim +++ b/tests/script/general/table/column_value.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/createmulti.sim b/tests/script/general/table/createmulti.sim new file mode 100644 index 0000000000000000000000000000000000000000..0da1ce96a759551936eb00b00d6915b01682a64c --- /dev/null +++ b/tests/script/general/table/createmulti.sim @@ -0,0 +1,46 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 3000 +sql connect + +print =============== create database +sql create database db +sql show databases +if $rows != 1 then + return -1 +endi + +print $data00 $data01 $data02 + +print =============== create super table +sql create table db.st1 (ts timestamp, i int) tags (j int) +sql create table db.st2 (ts timestamp, i int, j int) tags (t1 int, t2 int, t3 int) +sql show db.stables +if $rows != 2 then + return -1 +endi + +print $data00 $data01 $data02 + +print =============== create multiple child tables +sql create table db.ct1 using db.st1 tags(1) db.ct2 using db.st1 tags(2); + +sql show db.tables +if $rows != 2 then + return -1 +endi + +sql create table db.ct3 using db.st2 tags(1, 1, 1) db.ct4 using db.st2 tags(2, 2, 2); +sql show db.tables +if $rows != 4 then + return -1 +endi + +sql create table db.ct5 using db.st1 tags(3) db.ct6 using db.st2 tags(3, 3, 3); +sql show db.tables +if $rows != 6 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/table/date.sim b/tests/script/general/table/date.sim index 2e73a217a616859781f709cfa9c7117f6e34b9ae..23188e12e0a9b148877c87b67079c5429455eece 100644 --- a/tests/script/general/table/date.sim +++ b/tests/script/general/table/date.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/db.table.sim b/tests/script/general/table/db.table.sim index 50224f83b554001cd725926986d4769d36a47cb7..906396402a1abad594e08387878bf7c3674f55ad 100644 --- a/tests/script/general/table/db.table.sim +++ b/tests/script/general/table/db.table.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/delete_reuse1.sim b/tests/script/general/table/delete_reuse1.sim index 17974ccf4d3c2348c51b15e6382884d73fd73bf9..db594254873d47564e94927b04765c32798866bd 100644 --- a/tests/script/general/table/delete_reuse1.sim +++ b/tests/script/general/table/delete_reuse1.sim @@ -4,10 +4,10 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 @@ -22,7 +22,7 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/table/delete_reuse2.sim b/tests/script/general/table/delete_reuse2.sim index 917893b2d286208737693d10b8ec146455c8ed3c..f2784a3f5f2b3cb9ae9729b8ed254502eec7766a 100644 --- a/tests/script/general/table/delete_reuse2.sim +++ b/tests/script/general/table/delete_reuse2.sim @@ -4,10 +4,10 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 @@ -22,7 +22,7 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======== step1 diff --git a/tests/script/general/table/delete_writing.sim b/tests/script/general/table/delete_writing.sim index 2a7fb09104391c0bf7dd04ed94ca6e710c3d9bb2..5351d13d80ad732b2a58953f72b3c62086b1090b 100644 --- a/tests/script/general/table/delete_writing.sim +++ b/tests/script/general/table/delete_writing.sim @@ -4,10 +4,10 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 @@ -41,7 +41,7 @@ while $x < 15 sql create table db.tb (ts timestamp, i int) - sleep 3000 + sleep 1000 $x = $x + 1 endw diff --git a/tests/script/general/table/describe.sim b/tests/script/general/table/describe.sim index ebec004f1964119d97504f8bc78a0b42941824ab..e59371e530a51a171efa64393925e6d311230432 100644 --- a/tests/script/general/table/describe.sim +++ b/tests/script/general/table/describe.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/double.sim b/tests/script/general/table/double.sim index 10239568fe488de55961e134a8e248cb83963393..ab3f2428bde489b3024e332861bad1d38cb99179 100644 --- a/tests/script/general/table/double.sim +++ b/tests/script/general/table/double.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/fill.sim b/tests/script/general/table/fill.sim index 333573e577b636ef780e652d1dc61d4d6701f925..069eeff6cf0cdf35a55f33fa3e5bbf8135cc7921 100644 --- a/tests/script/general/table/fill.sim +++ b/tests/script/general/table/fill.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =================== step1 @@ -42,9 +42,9 @@ sql select count(*), last(ts), min(k), max(k), avg(k) from db.mt where a=0 and t print =================== step2 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 -system sh/exec.sh -n dnode1 -s start sleep 3000 +system sh/exec.sh -n dnode1 -s start +sleep 2000 print =================== step3 sql select * from db.mt diff --git a/tests/script/general/table/float.sim b/tests/script/general/table/float.sim index e4ef8a42d67569aa75dda90236861ac11c951870..2d0ea0e5eab06ad127ef445d140c0cd52a50532e 100644 --- a/tests/script/general/table/float.sim +++ b/tests/script/general/table/float.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/int.sim b/tests/script/general/table/int.sim index 81bdcda47dd14c59df6017a07dde64da87b6546f..f30b5b28f5457ccfbf83d48816f2417cbd3e3d46 100644 --- a/tests/script/general/table/int.sim +++ b/tests/script/general/table/int.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/limit.sim b/tests/script/general/table/limit.sim index 18597f2e1cd8ebed7747fed32067ed51664aebc8..dd38453d0cb0580e446992ae36f022e5e2ac1185 100644 --- a/tests/script/general/table/limit.sim +++ b/tests/script/general/table/limit.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 129 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 8 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/table/smallint.sim b/tests/script/general/table/smallint.sim index b6f0d6948faf9323085902cfc9b463238cb3f9ca..f622ce7853062edb15dca4344101cd6a96299225 100644 --- a/tests/script/general/table/smallint.sim +++ b/tests/script/general/table/smallint.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/table.sim b/tests/script/general/table/table.sim index 6ad1b13b1cae6cd05b3b0bd3de4c648e4aab1fde..c9806c40c649695a489bdcc659e085b03f7d674b 100644 --- a/tests/script/general/table/table.sim +++ b/tests/script/general/table/table.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/table/table_len.sim b/tests/script/general/table/table_len.sim index 2568ebc7a5e460284f8a4d620beffadd73e76baf..d95c9ab0aa9cdc4db66d65d2334088550c05bfa9 100644 --- a/tests/script/general/table/table_len.sim +++ b/tests/script/general/table/table_len.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/tinyint.sim b/tests/script/general/table/tinyint.sim index 017c007e8425385c6045d8c4ea1d3f7335e62a0d..afa931fc79802e5790f90492576c90acb51e06dc 100644 --- a/tests/script/general/table/tinyint.sim +++ b/tests/script/general/table/tinyint.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/table/vgroup.sim b/tests/script/general/table/vgroup.sim index f4496e2f1988f86fc756dddc124224e84ed6fc06..d306a4731db2023e87ac0e69378e448bbd555c56 100644 --- a/tests/script/general/table/vgroup.sim +++ b/tests/script/general/table/vgroup.sim @@ -1,11 +1,11 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/general/tag/3.sim b/tests/script/general/tag/3.sim index 4b3104fe1bf591b69e4cc8837df9007ed2e815a5..20185f5f01403b838799427d9ac39fcafe653d86 100644 --- a/tests/script/general/tag/3.sim +++ b/tests/script/general/tag/3.sim @@ -1,9 +1,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/4.sim b/tests/script/general/tag/4.sim index 120ef4e4e3ba4dd5446eff7143d2dd605374fa5d..ee3c8efa6c47be1fc91b1114294b6ebf832eabdc 100644 --- a/tests/script/general/tag/4.sim +++ b/tests/script/general/tag/4.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql reset query cache diff --git a/tests/script/general/tag/5.sim b/tests/script/general/tag/5.sim index efe88b36503c5161179602afea18fcf6ea1a106a..895b1a9492644290e5c30abc901849e1adf33e6c 100644 --- a/tests/script/general/tag/5.sim +++ b/tests/script/general/tag/5.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql reset query cache diff --git a/tests/script/general/tag/6.sim b/tests/script/general/tag/6.sim index d5f0a6c4af9b13420a07d4140a456f2b5112acd2..9190998bb316bc63fd47022029b4c8deb61c0727 100644 --- a/tests/script/general/tag/6.sim +++ b/tests/script/general/tag/6.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql reset query cache diff --git a/tests/script/general/tag/add.sim b/tests/script/general/tag/add.sim index 301ec4f825c9954f9b5790c5e27e417538c3b4e2..4a3871235ecfbd1546cbab1ba4ab90de33ac7ed8 100644 --- a/tests/script/general/tag/add.sim +++ b/tests/script/general/tag/add.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/bigint.sim b/tests/script/general/tag/bigint.sim index 4fa22d3995617ec16d9ad85d5ef9a11bfe65ac06..3e5d528980e3f8716ae5dea0c1d4090e02d0420a 100644 --- a/tests/script/general/tag/bigint.sim +++ b/tests/script/general/tag/bigint.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/binary.sim b/tests/script/general/tag/binary.sim index 7c6f95b1ef785d5621d02e51e28fe96f7fe37f6c..960f45675d28349d170544729055ca3b815a2ccb 100644 --- a/tests/script/general/tag/binary.sim +++ b/tests/script/general/tag/binary.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/binary_binary.sim b/tests/script/general/tag/binary_binary.sim index 077ec85b523d9fa20a90c1ec8f0e9309dd2b3ea2..3a0fb568482fa19a142f4b9019bd09c5208b4252 100644 --- a/tests/script/general/tag/binary_binary.sim +++ b/tests/script/general/tag/binary_binary.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/bool.sim b/tests/script/general/tag/bool.sim index 349cb738bffbb216305af0718611987ae7f20f9c..e37cba669ba27ec4046e8483938763b90360de31 100644 --- a/tests/script/general/tag/bool.sim +++ b/tests/script/general/tag/bool.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/bool_binary.sim b/tests/script/general/tag/bool_binary.sim index a3f99e25ad8f62562cc70f3776e99397a815ae2f..9f6e4f734432bd39c6250adbc0fe26883f104267 100644 --- a/tests/script/general/tag/bool_binary.sim +++ b/tests/script/general/tag/bool_binary.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/bool_int.sim b/tests/script/general/tag/bool_int.sim index 1a2726dbaa684e2c39a37f1ef29827b480fc3617..60345c2d68b9216acc78e8bb8617c698013f4402 100644 --- a/tests/script/general/tag/bool_int.sim +++ b/tests/script/general/tag/bool_int.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/change.sim b/tests/script/general/tag/change.sim index a7b8554bd96f6e8857d9b6a6b2de4faa4459bf2e..6f294c0f48fb18d232a5e78f9fbec084e8dc699d 100644 --- a/tests/script/general/tag/change.sim +++ b/tests/script/general/tag/change.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -175,7 +175,7 @@ sql alter table $mt change tag tgcol3 tgcol9 sql alter table $mt change tag tgcol5 tgcol10 sql alter table $mt change tag tgcol6 tgcol11 -sleep 5000 +sleep 3000 sql reset query cache print =============== step2 diff --git a/tests/script/general/tag/column.sim b/tests/script/general/tag/column.sim index 4a36e832f1bf6f373a81318e483ee02ee6bf2e01..5a0cd169c5a77f0dfd6f81b615bbd09bb8e5a629 100644 --- a/tests/script/general/tag/column.sim +++ b/tests/script/general/tag/column.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/commit.sim b/tests/script/general/tag/commit.sim index 8b07d98afbaf343056ca94955e1b8977e1c46fe1..bcd9d7c618f68eb356813224a7b75f6e649a2e12 100644 --- a/tests/script/general/tag/commit.sim +++ b/tests/script/general/tag/commit.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -821,9 +821,9 @@ if $data07 != 12 then endi system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 print =============== step1 $i = 0 diff --git a/tests/script/general/tag/create.sim b/tests/script/general/tag/create.sim index adbb14e88aa28a1b5b1d1f7cd7bdab53c6d7c104..95b416654394946287cc8f9cdb95b984edbf3a88 100644 --- a/tests/script/general/tag/create.sim +++ b/tests/script/general/tag/create.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/delete.sim b/tests/script/general/tag/delete.sim index f3d0735b5c2c158b1bbc9725755c7349a474178a..2a0aa27bde6ef70e5354695a83dfc2d3853e1463 100644 --- a/tests/script/general/tag/delete.sim +++ b/tests/script/general/tag/delete.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start @@ -409,7 +409,7 @@ sql alter table $mt drop tag tgcol3 sql alter table $mt drop tag tgcol4 sql alter table $mt drop tag tgcol6 -sleep 5000 +sleep 3000 print =============== step2 $i = 2 diff --git a/tests/script/general/tag/double.sim b/tests/script/general/tag/double.sim index 13dcb1fc72b7c760c325a09fe0589ce0a211e23a..f17043393ffe51f7cdce4fb053168746430ab498 100644 --- a/tests/script/general/tag/double.sim +++ b/tests/script/general/tag/double.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/filter.sim b/tests/script/general/tag/filter.sim index 75a6ed00da6d3b8173b748961d42ab8f909e7ef1..7a899a7e67d9261ea2a68c5ad1ee311f03278cb9 100644 --- a/tests/script/general/tag/filter.sim +++ b/tests/script/general/tag/filter.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/float.sim b/tests/script/general/tag/float.sim index 2352bbaf2e8a2c5e27158332434d59be62e8d9d6..35bf7d6090140ba73cb0c63efd57d828fb6b21bb 100644 --- a/tests/script/general/tag/float.sim +++ b/tests/script/general/tag/float.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/int.sim b/tests/script/general/tag/int.sim index a93df9a57afabdd59d22e1de44504db1bdbdaef4..4eea02c9cd4438f16cdf759fef0ade88eaabc8b1 100644 --- a/tests/script/general/tag/int.sim +++ b/tests/script/general/tag/int.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/int_binary.sim b/tests/script/general/tag/int_binary.sim index d3795006680a771190930b446c2390a5240dcb01..9a756976769c6294f65f61c5318033274b1145f2 100644 --- a/tests/script/general/tag/int_binary.sim +++ b/tests/script/general/tag/int_binary.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/int_float.sim b/tests/script/general/tag/int_float.sim index 510c4f92d0810f6a675f0b0180081c0bf552584e..a03c4b7148ab14c9c0b1e28d3fc28dc86df0c2d9 100644 --- a/tests/script/general/tag/int_float.sim +++ b/tests/script/general/tag/int_float.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/set.sim b/tests/script/general/tag/set.sim index 112695d37b0a924779812485e22ffb5680a8856e..cbc964fad7d73b9ba6fee5128e50bb4b9dfa433b 100644 --- a/tests/script/general/tag/set.sim +++ b/tests/script/general/tag/set.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c walLevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/smallint.sim b/tests/script/general/tag/smallint.sim index 18b0957b1517560bca651ef9fbd4cc9325e94a22..e69eef05f274175b55ba78e4c9f69a5e102eb6f8 100644 --- a/tests/script/general/tag/smallint.sim +++ b/tests/script/general/tag/smallint.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/tag/tinyint.sim b/tests/script/general/tag/tinyint.sim index f104f8df04356487999bd4232db70b9ea067dd73..2d70dc7d4880d4b1540da557f779fe06b877dabc 100644 --- a/tests/script/general/tag/tinyint.sim +++ b/tests/script/general/tag/tinyint.sim @@ -2,10 +2,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ======================== dnode1 start diff --git a/tests/script/general/user/authority.sim b/tests/script/general/user/authority.sim index 71f185043b3ef8fbe0fb09d53c9d4dae372d41fa..45230e3e8af59dcac57ff71a7802ba6a637cc8d7 100644 --- a/tests/script/general/user/authority.sim +++ b/tests/script/general/user/authority.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ============= step1 sql create user read pass 'taosdata' diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim index eb543612f560fa43be7cb8a87ba36c3a344c1212..90aad59932115e04263da8cc1e8b91effb09eb07 100644 --- a/tests/script/general/user/monitor.sim +++ b/tests/script/general/user/monitor.sim @@ -2,12 +2,12 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 print ========== step1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c monitor -v 1 system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ========== step2 @@ -15,18 +15,18 @@ print ========== step2 # return -1 #step21: sql drop table log.dn -x step22 - return -1 +# return -1 step22: sql drop user log -x step23 - return -1 +# return -1 step23: print ========== step3 -sleep 3000 -sql select * from log.dn -if $rows == 0 then - return -1 -endi +sleep 2000 +#sql select * from log.dn +#if $rows == 0 then +# return -1 +#endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/user/pass_alter.sim b/tests/script/general/user/pass_alter.sim index 857d658db1ee7f1b91220106983ebb95c9b687d6..964e311ec0b17eb99d8cbfd745cc4510b9a94252 100644 --- a/tests/script/general/user/pass_alter.sim +++ b/tests/script/general/user/pass_alter.sim @@ -2,7 +2,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/general/user/pass_len.sim b/tests/script/general/user/pass_len.sim index d5d7b3250fe18490f92a299d100990f8b0405bca..5eb200b51f9ac28b19ea095be6530dbada9be894 100644 --- a/tests/script/general/user/pass_len.sim +++ b/tests/script/general/user/pass_len.sim @@ -4,7 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/user/user_len.sim b/tests/script/general/user/user_len.sim index 79b72468bbf64a5f834c34979391f43c25809c0c..55e5eb19ef476691882e7c9ba589697aef8190da 100644 --- a/tests/script/general/user/user_len.sim +++ b/tests/script/general/user/user_len.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/general/vector/metrics_field.sim b/tests/script/general/vector/metrics_field.sim index 2a03ccea19a46771a89755791c7b696f547d296c..2719805c63af87d4088a9b1fd87249f09e51b957 100644 --- a/tests/script/general/vector/metrics_field.sim +++ b/tests/script/general/vector/metrics_field.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mf_db diff --git a/tests/script/general/vector/metrics_mix.sim b/tests/script/general/vector/metrics_mix.sim index 96b2f6a7a013d4bd323de7db9c39dfcf433421b2..7c9bb3b668c67fbc75cffc9b15bad1b276142917 100644 --- a/tests/script/general/vector/metrics_mix.sim +++ b/tests/script/general/vector/metrics_mix.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mx_db diff --git a/tests/script/general/vector/metrics_query.sim b/tests/script/general/vector/metrics_query.sim index 824aa6f22e45153b295f2b42afe9ebcf45d00642..fd635a3104f70f4fcbcd71914b476dfc3fb69ea8 100644 --- a/tests/script/general/vector/metrics_query.sim +++ b/tests/script/general/vector/metrics_query.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mq_db diff --git a/tests/script/general/vector/metrics_tag.sim b/tests/script/general/vector/metrics_tag.sim index e7575cc9df5e9c0c49af8ec21ed629c73cd7a0e6..1d412d35d3669ced6437e6025f8415ddd3a1cf1e 100644 --- a/tests/script/general/vector/metrics_tag.sim +++ b/tests/script/general/vector/metrics_tag.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mtg_db diff --git a/tests/script/general/vector/metrics_time.sim b/tests/script/general/vector/metrics_time.sim index bbcabfd1b7f81dc65f8c61f4c46d09bd18d1c35b..d0152439bff2c9ab5450d870b2238d0b137a2fa4 100644 --- a/tests/script/general/vector/metrics_time.sim +++ b/tests/script/general/vector/metrics_time.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mt_db diff --git a/tests/script/general/vector/multi.sim b/tests/script/general/vector/multi.sim index 105f210950960ac8dc75dd38e22af1fb8dd5b060..1101b0b0dbc4484e8ccef3aaab5ce8fe54b03d0b 100644 --- a/tests/script/general/vector/multi.sim +++ b/tests/script/general/vector/multi.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_mu_db diff --git a/tests/script/general/vector/single.sim b/tests/script/general/vector/single.sim index 2292d6f44ae43286d619a560320c31b3085ea267..e979a0ffb71e78ed2a3a79cb32e5b34ab98fd21a 100644 --- a/tests/script/general/vector/single.sim +++ b/tests/script/general/vector/single.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_si_db diff --git a/tests/script/general/vector/table_field.sim b/tests/script/general/vector/table_field.sim index a9d6910f4da4c1258d7a2efdf592352e7d043349..d86eb99331f9c65c013d4d9bdd5798672629eab9 100644 --- a/tests/script/general/vector/table_field.sim +++ b/tests/script/general/vector/table_field.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_tf_db diff --git a/tests/script/general/vector/table_mix.sim b/tests/script/general/vector/table_mix.sim index a6c4dc92da51d312c2a65561ecb8fcbf4737239a..5c4fb52888d3a02608bdc90d4c213c3a4f4ac0ff 100644 --- a/tests/script/general/vector/table_mix.sim +++ b/tests/script/general/vector/table_mix.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_tm_db diff --git a/tests/script/general/vector/table_query.sim b/tests/script/general/vector/table_query.sim index 812229afc4acaf7cb8515c3cbb483bad302b8ac0..9ef18255a9582982cddcd8c8c699adeb6f01f1b6 100644 --- a/tests/script/general/vector/table_query.sim +++ b/tests/script/general/vector/table_query.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_tq_db diff --git a/tests/script/general/vector/table_time.sim b/tests/script/general/vector/table_time.sim index a83195beba2fe0dec305098407ec872aa47e8535..c38546b1170fdbe5249a4220fbf112ea8eb25b18 100644 --- a/tests/script/general/vector/table_time.sim +++ b/tests/script/general/vector/table_time.sim @@ -1,10 +1,10 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $dbPrefix = m_tt_db diff --git a/tests/script/general/wal/kill.sim b/tests/script/general/wal/kill.sim index 7f103874a561c2bb2534996ab30d30ab0e8907a3..94a35b636e7915842787ed982590b39a93e1f408 100644 --- a/tests/script/general/wal/kill.sim +++ b/tests/script/general/wal/kill.sim @@ -13,62 +13,62 @@ sql create table t1 (ts timestamp, i int) sql insert into t1 values(now, 1); print =============== step3 -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step4 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step5 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step6 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step7 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then return -1 endi system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sleep 3000 +sleep 2000 print =============== step8 system sh/exec.sh -n dnode1 -s start -x SIGKILL -sleep 3000 +sleep 2000 sql select * from t1; print rows: $rows if $rows != 1 then diff --git a/tests/script/general/wal/maxtables.sim b/tests/script/general/wal/maxtables.sim index e504c7e92e3447f7d29dbb2dc03456c3775ced2c..acd6af1d1a1dbb02a3aef35fe2d9c07cae74b529 100644 --- a/tests/script/general/wal/maxtables.sim +++ b/tests/script/general/wal/maxtables.sim @@ -32,11 +32,11 @@ endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 -sleep 3000 +sleep 2000 print =============== step4 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql select * from st; if $rows != 100 then diff --git a/tests/script/issue/TD-3300.sim b/tests/script/issue/TD-3300.sim new file mode 100644 index 0000000000000000000000000000000000000000..0745ceb8490567b508397b3e99d4748dc41c8971 --- /dev/null +++ b/tests/script/issue/TD-3300.sim @@ -0,0 +1,556 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 + +system sh/cfg.sh -n dnode1 -c role -v 1 +system sh/cfg.sh -n dnode2 -c role -v 2 +system sh/cfg.sh -n dnode3 -c role -v 2 +system sh/cfg.sh -n dnode4 -c role -v 2 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator + +print ============== step0: start tarbitrator +system sh/exec_tarbitrator.sh -s start + +print ============== step1: start dnode1, only deploy mnode +system sh/exec.sh -n dnode1 -s start +sql connect + +print ============== step2: start dnode2/dnode3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname2 +sql create dnode $hostname3 + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step2 +endi +if $data4_2 != ready then + goto step2 +endi +if $data4_3 != ready then + goto step2 +endi + +sleep 1000 + +print ============== step3 +sql create database db replica 2 +sql use db + +sql create table stb (ts timestamp, c1 int, c2 int) tags(t1 int) +sql create table t1 using stb tags(1) +sql insert into t1 values(1577980800000, 1, 5) +sql insert into t1 values(1577980800001, 2, 4) +sql insert into t1 values(1577980800002, 3, 3) +sql insert into t1 values(1577980800003, 4, 2) +sql insert into t1 values(1577980800004, 5, 1) + +sql show db.vgroups +if $data04 != 3 then + return -1 +endi +if $data06 != 2 then + return -1 +endi +if $data05 != master then + return -1 +endi +if $data07 != slave then + return -1 +endi + +sql select * from t1 +if $rows != 5 then + return -1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGKILL +system sh/exec.sh -n dnode3 -s stop -x SIGKILL + +print ============== step4 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step4 +endi +if $data4_2 != ready then + goto step4 +endi +if $data4_3 != ready then + goto step4 +endi + +sql show db.vgroups +if $data04 != 3 then + goto step4 +endi +if $data06 != 2 then + goto step4 +endi +if $data05 != master then + goto step4 +endi +if $data07 != slave then + goto step4 +endi + +sql create table t2 using stb tags(1) +sql insert into t2 values(1577980800000, 1, 5) +sql insert into t2 values(1577980800001, 2, 4) +sql insert into t2 values(1577980800002, 3, 3) +sql insert into t2 values(1577980800003, 4, 2) +sql insert into t2 values(1577980800004, 5, 1) + +sql select * from t2 +if $rows != 5 then + return -1 +endi + +print ============== step5 +system sh/exec.sh -n dnode3 -s stop -x SIGKILL + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step5 +endi +if $data4_2 != ready then + goto step5 +endi +if $data4_3 != offline then + goto step5 +endi + +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi + +sql show db.vgroups +if $data04 != 3 then + goto step5 +endi +if $data06 != 2 then + goto step5 +endi +if $data05 != offline then + goto step5 +endi +if $data07 != master then + goto step5 +endi + +print ============== step6 +sql create table t3 using stb tags(1) +sql insert into t3 values(1577980800000, 1, 5) +sql insert into t3 values(1577980800001, 2, 4) +sql insert into t3 values(1577980800002, 3, 3) +sql insert into t3 values(1577980800003, 4, 2) +sql insert into t3 values(1577980800004, 5, 1) +sql insert into t3 values(1577980800010, 11, 5) +sql insert into t3 values(1577980800011, 12, 4) +sql insert into t3 values(1577980800012, 13, 3) +sql insert into t3 values(1577980800013, 14, 2) +sql insert into t3 values(1577980800014, 15, 1) + +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi +sql select * from t3 +if $rows != 10 then + return -1 +endi + +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step6 +endi +if $data4_2 != ready then + goto step6 +endi +if $data4_3 != ready then + goto step6 +endi + +sql show db.vgroups +if $data04 != 3 then + goto step6 +endi +if $data06 != 2 then + goto step6 +endi +if $data05 != slave then + goto step6 +endi +if $data07 != master then + goto step6 +endi + +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi +sql select * from t3 +if $rows != 10 then + return -1 +endi + +print ============== step7 +sql create table t4 using stb tags(1) +sql insert into t4 values(1577980800000, 1, 5) +sql insert into t4 values(1577980800001, 2, 4) +sql insert into t4 values(1577980800002, 3, 3) +sql insert into t4 values(1577980800003, 4, 2) +sql insert into t4 values(1577980800004, 5, 1) +sql insert into t4 values(1577980800010, 11, 5) +sql insert into t4 values(1577980800011, 12, 4) +sql insert into t4 values(1577980800012, 13, 3) +sql insert into t4 values(1577980800013, 14, 2) +sql insert into t4 values(1577980800014, 15, 1) +sql insert into t4 values(1577980800020, 21, 5) +sql insert into t4 values(1577980800021, 22, 4) +sql insert into t4 values(1577980800022, 23, 3) +sql insert into t4 values(1577980800023, 24, 2) +sql insert into t4 values(1577980800024, 25, 1) + +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi +sql select * from t3 +if $rows != 10 then + return -1 +endi +sql select * from t4 +if $rows != 15 then + return -1 +endi + +system sh/exec.sh -n dnode2 -s stop -x SIGKILL +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step7 +endi +if $data4_2 != offline then + goto step7 +endi +if $data4_3 != ready then + goto step7 +endi + +sql show db.vgroups +if $data04 != 3 then + goto step7 +endi +if $data06 != 2 then + goto step7 +endi +if $data05 != master then + goto step7 +endi +if $data07 != offline then + goto step7 +endi + +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi +sql select * from t3 +if $rows != 10 then + return -1 +endi +sql select * from t4 +if $rows != 15 then + return -1 +endi + +print ============== step8 +sql create table t5 using stb tags(1) +sql insert into t5 values(1577980800000, 1, 5) +sql insert into t5 values(1577980800001, 2, 4) +sql insert into t5 values(1577980800002, 3, 3) +sql insert into t5 values(1577980800003, 4, 2) +sql insert into t5 values(1577980800004, 5, 1) +sql insert into t5 values(1577980800010, 11, 5) + +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi +sql select * from t3 +if $rows != 10 then + return -1 +endi +sql select * from t4 +if $rows != 15 then + return -1 +endi +sql select * from t5 +if $rows != 6 then + return -1 +endi + +system sh/exec.sh -n dnode2 -s start +$x = 0 +step8: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step8 +endi +if $data4_2 != ready then + goto step8 +endi +if $data4_3 != ready then + goto step8 +endi + +sql show db.vgroups +if $data04 != 3 then + goto step8 +endi +if $data06 != 2 then + goto step8 +endi +if $data05 != master then + goto step8 +endi +if $data07 != slave then + goto step8 +endi + +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi +sql select * from t3 +if $rows != 10 then + return -1 +endi +sql select * from t4 +if $rows != 15 then + return -1 +endi +sql select * from t5 +if $rows != 6 then + return -1 +endi + +print ============== step9 +sql create table t6 using stb tags(1) +sql insert into t6 values(1577980800000, 1, 5) +sql insert into t6 values(1577980800001, 2, 4) +sql insert into t6 values(1577980800002, 3, 3) +sql insert into t6 values(1577980800003, 4, 2) +sql insert into t6 values(1577980800004, 5, 1) +sql insert into t6 values(1577980800010, 11, 5) +sql insert into t6 values(1577980800011, 12, 4) + +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi +sql select * from t3 +if $rows != 10 then + return -1 +endi +sql select * from t4 +if $rows != 15 then + return -1 +endi +sql select * from t5 +if $rows != 6 then + return -1 +endi +sql select * from t6 +if $rows != 7 then + return -1 +endi + +system sh/exec.sh -n dnode3 -s stop -x SIGKILL +$x = 0 +step9: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 $data4_1 +print dnode2 $data4_2 +print dnode3 $data4_3 + +if $data4_1 != ready then + goto step9 +endi +if $data4_2 != ready then + goto step9 +endi +if $data4_3 != offline then + goto step9 +endi + +print ============== 2 +sql show db.vgroups + +if $data04 != 3 then + goto step7 +endi +if $data06 != 2 then + goto step7 +endi +if $data05 != offline then + goto step7 +endi +if $data07 != master then + goto step7 +endi + +print ============== 3 +sql select * from t1 +if $rows != 5 then + return -1 +endi +sql select * from t2 +if $rows != 5 then + return -1 +endi +sql select * from t3 +if $rows != 10 then + return -1 +endi +sql select * from t4 +if $rows != 15 then + return -1 +endi +sql select * from t5 +if $rows != 6 then + return -1 +endi +sql select * from t6 +if $rows != 7 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop +system sh/exec.sh -n dnode2 -s stop +system sh/exec.sh -n dnode3 -s stop diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 90d01b7215cc559a3c543c5b46c80e1518a3acdb..ce770aee56c9ddb7b3a6b866e4317d09c463c8d6 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -1,74 +1,7 @@ cd ../../../debug; cmake .. cd ../../../debug; make -./test.sh -f issue/TD-2677.sim -./test.sh -f issue/TD-2680.sim -./test.sh -f issue/TD-2713.sim - -./test.sh -f general/alter/cached_schema_after_alter.sim -./test.sh -f general/alter/count.sim -./test.sh -f general/alter/dnode.sim -./test.sh -f general/alter/import.sim -./test.sh -f general/alter/insert1.sim -./test.sh -f general/alter/insert2.sim -./test.sh -f general/alter/metrics.sim -./test.sh -f general/alter/table.sim - -./test.sh -f general/cache/new_metrics.sim -./test.sh -f general/cache/restart_metrics.sim -./test.sh -f general/cache/restart_table.sim - -./test.sh -f general/connection/connection.sim - -./test.sh -f general/column/commit.sim -./test.sh -f general/column/metrics.sim -./test.sh -f general/column/table.sim - -./test.sh -f general/compress/commitlog.sim -./test.sh -f general/compress/compress.sim -./test.sh -f general/compress/compress2.sim -./test.sh -f general/compress/uncompress.sim - -./test.sh -f general/compute/avg.sim -./test.sh -f general/compute/bottom.sim -./test.sh -f general/compute/count.sim -./test.sh -f general/compute/diff.sim -./test.sh -f general/compute/diff2.sim -./test.sh -f general/compute/first.sim -./test.sh -f general/compute/interval.sim -./test.sh -f general/compute/last.sim -./test.sh -f general/compute/leastsquare.sim -./test.sh -f general/compute/max.sim -./test.sh -f general/compute/min.sim -./test.sh -f general/compute/null.sim -./test.sh -f general/compute/percentile.sim -./test.sh -f general/compute/stddev.sim -./test.sh -f general/compute/sum.sim -./test.sh -f general/compute/top.sim - -./test.sh -f general/db/alter_option.sim -./test.sh -f general/db/alter_tables_d2.sim -./test.sh -f general/db/alter_tables_v1.sim -./test.sh -f general/db/alter_tables_v4.sim -./test.sh -f general/db/alter_vgroups.sim -./test.sh -f general/db/basic.sim -./test.sh -f general/db/basic1.sim -./test.sh -f general/db/basic2.sim -./test.sh -f general/db/basic3.sim -./test.sh -f general/db/basic4.sim -./test.sh -f general/db/basic5.sim -./test.sh -f general/db/delete_reuse1.sim -./test.sh -f general/db/delete_reuse2.sim -./test.sh -f general/db/delete_reusevnode.sim -./test.sh -f general/db/delete_reusevnode2.sim -./test.sh -f general/db/delete_writing1.sim -./test.sh -f general/db/delete_writing2.sim -./test.sh -f general/db/delete.sim -./test.sh -f general/db/len.sim -./test.sh -f general/db/repeat.sim -./test.sh -f general/db/tables.sim -./test.sh -f general/db/vnodes.sim -./test.sh -f general/db/nosuchfile.sim +#======================b1-start=============== ./test.sh -f general/field/2.sim ./test.sh -f general/field/3.sim @@ -94,11 +27,6 @@ cd ../../../debug; make ./test.sh -f general/http/grafana_bug.sim ./test.sh -f general/http/grafana.sim -./test.sh -f general/import/basic.sim -./test.sh -f general/import/commit.sim -./test.sh -f general/import/large.sim -./test.sh -f general/import/replica1.sim - ./test.sh -f general/insert/basic.sim ./test.sh -f general/insert/insert_drop.sim ./test.sh -f general/insert/query_block1_memory.sim @@ -158,43 +86,12 @@ cd ../../../debug; make ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim +./test.sh -f general/db/nosuchfile.sim ./test.sh -f general/parser/function.sim +./test.sh -f unique/cluster/vgroup100.sim -./test.sh -f general/stable/disk.sim -./test.sh -f general/stable/dnode3.sim -./test.sh -f general/stable/metrics.sim -./test.sh -f general/stable/refcount.sim -./test.sh -f general/stable/show.sim -./test.sh -f general/stable/values.sim -./test.sh -f general/stable/vnode3.sim - -./test.sh -f general/table/autocreate.sim -./test.sh -f general/table/basic1.sim -./test.sh -f general/table/basic2.sim -./test.sh -f general/table/basic3.sim -./test.sh -f general/table/bigint.sim -./test.sh -f general/table/binary.sim -./test.sh -f general/table/bool.sim -./test.sh -f general/table/column_name.sim -./test.sh -f general/table/column_num.sim -./test.sh -f general/table/column_value.sim -./test.sh -f general/table/column2.sim -./test.sh -f general/table/date.sim -./test.sh -f general/table/db.table.sim -./test.sh -f general/table/delete_reuse1.sim -./test.sh -f general/table/delete_reuse2.sim -./test.sh -f general/table/delete_writing.sim -./test.sh -f general/table/describe.sim -./test.sh -f general/table/double.sim -./test.sh -f general/table/fill.sim -./test.sh -f general/table/float.sim -./test.sh -f general/table/int.sim -./test.sh -f general/table/limit.sim -./test.sh -f general/table/smallint.sim -./test.sh -f general/table/table_len.sim -./test.sh -f general/table/table.sim -./test.sh -f general/table/tinyint.sim -./test.sh -f general/table/vgroup.sim +#======================b1-end=============== +#======================b2-start=============== ./test.sh -f general/tag/3.sim ./test.sh -f general/tag/4.sim @@ -221,6 +118,9 @@ cd ../../../debug; make ./test.sh -f general/tag/set.sim ./test.sh -f general/tag/smallint.sim ./test.sh -f general/tag/tinyint.sim +./test.sh -f general/wal/sync.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim ./test.sh -f general/user/authority.sim ./test.sh -f general/user/monitor.sim @@ -241,10 +141,6 @@ cd ../../../debug; make ./test.sh -f general/vector/table_query.sim ./test.sh -f general/vector/table_time.sim -./test.sh -f general/wal/sync.sim -./test.sh -f general/wal/kill.sim -./test.sh -f general/wal/maxtables.sim - ./test.sh -f unique/account/account_create.sim ./test.sh -f unique/account/account_delete.sim ./test.sh -f unique/account/account_len.sim @@ -257,91 +153,15 @@ cd ../../../debug; make ./test.sh -f unique/account/user_create.sim ./test.sh -f unique/account/user_len.sim -./test.sh -f unique/big/balance.sim ./test.sh -f unique/big/maxvnodes.sim ./test.sh -f unique/big/tcp.sim ./test.sh -f unique/cluster/alter.sim -./test.sh -f unique/cluster/balance1.sim -./test.sh -f unique/cluster/balance2.sim -./test.sh -f unique/cluster/balance3.sim ./test.sh -f unique/cluster/cache.sim -./test.sh -f unique/cluster/vgroup100.sim - -./test.sh -f unique/column/replica3.sim - -./test.sh -f unique/db/commit.sim -./test.sh -f unique/db/delete.sim -./test.sh -f unique/db/delete_part.sim -./test.sh -f unique/db/replica_add12.sim -./test.sh -f unique/db/replica_add13.sim -./test.sh -f unique/db/replica_add23.sim -./test.sh -f unique/db/replica_reduce21.sim -./test.sh -f unique/db/replica_reduce32.sim -./test.sh -f unique/db/replica_reduce31.sim -./test.sh -f unique/db/replica_part.sim - -./test.sh -f unique/dnode/alternativeRole.sim -./test.sh -f unique/dnode/monitor.sim -./test.sh -f unique/dnode/monitor_bug.sim -./test.sh -f unique/dnode/simple.sim -./test.sh -f unique/dnode/balance1.sim -./test.sh -f unique/dnode/balance2.sim -./test.sh -f unique/dnode/balance3.sim -./test.sh -f unique/dnode/balancex.sim -./test.sh -f unique/dnode/data1.sim -./test.sh -f unique/dnode/m2.sim -./test.sh -f unique/dnode/m3.sim -./test.sh -f unique/dnode/lossdata.sim -./test.sh -f unique/dnode/offline1.sim -./test.sh -f unique/dnode/offline2.sim -./test.sh -f unique/dnode/offline3.sim -./test.sh -f unique/dnode/reason.sim -./test.sh -f unique/dnode/remove1.sim -./test.sh -f unique/dnode/remove2.sim -./test.sh -f unique/dnode/vnode_clean.sim -./test.sh -f unique/http/admin.sim -./test.sh -f unique/http/opentsdb.sim -./test.sh -f unique/import/replica2.sim -./test.sh -f unique/import/replica3.sim - -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -./test.sh -f unique/mnode/mgmt25.sim -./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt30.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - -./test.sh -f unique/vnode/many.sim -./test.sh -f unique/vnode/replica2_basic2.sim -./test.sh -f unique/vnode/replica2_repeat.sim -./test.sh -f unique/vnode/replica3_basic.sim -./test.sh -f unique/vnode/replica3_repeat.sim -./test.sh -f unique/vnode/replica3_vgroup.sim - -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_replica1_vnoden.sim +#======================b2-end=============== +#======================b3-start=============== ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim @@ -363,7 +183,7 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim ./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim -./test.sh -f unique/arbitrator/insert_duplicationTs.sim + ./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim ./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim ./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim @@ -390,7 +210,220 @@ cd ../../../debug; make ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +./test.sh -f unique/stable/balance_replica1.sim +./test.sh -f unique/stable/dnode2_stop.sim +./test.sh -f unique/stable/dnode2.sim +./test.sh -f unique/stable/dnode3.sim +./test.sh -f unique/stable/replica2_dnode4.sim +./test.sh -f unique/stable/replica2_vnode3.sim +./test.sh -f unique/stable/replica3_dnode6.sim +./test.sh -f unique/stable/replica3_vnode3.sim + +#======================b3-end=============== +#======================b4-start=============== + +./test.sh -f unique/http/admin.sim +./test.sh -f unique/http/opentsdb.sim + +./test.sh -f unique/import/replica2.sim +./test.sh -f unique/import/replica3.sim + +./test.sh -f general/alter/cached_schema_after_alter.sim +./test.sh -f general/alter/count.sim +./test.sh -f general/alter/dnode.sim +./test.sh -f general/alter/import.sim +./test.sh -f general/alter/insert1.sim +./test.sh -f general/alter/insert2.sim +./test.sh -f general/alter/metrics.sim +./test.sh -f general/alter/table.sim + +./test.sh -f general/cache/new_metrics.sim +./test.sh -f general/cache/restart_metrics.sim +./test.sh -f general/cache/restart_table.sim + +./test.sh -f general/connection/connection.sim + +./test.sh -f general/column/commit.sim +./test.sh -f general/column/metrics.sim +./test.sh -f general/column/table.sim + +./test.sh -f general/compress/commitlog.sim +./test.sh -f general/compress/compress.sim +./test.sh -f general/compress/compress2.sim +./test.sh -f general/compress/uncompress.sim + +./test.sh -f general/stable/disk.sim +./test.sh -f general/stable/dnode3.sim +./test.sh -f general/stable/metrics.sim +./test.sh -f general/stable/refcount.sim +./test.sh -f general/stable/show.sim +./test.sh -f general/stable/values.sim +./test.sh -f general/stable/vnode3.sim + +./test.sh -f unique/column/replica3.sim +./test.sh -f issue/TD-2713.sim +./test.sh -f general/parser/select_distinct_tag.sim +./test.sh -f unique/mnode/mgmt30.sim +./test.sh -f issue/TD-2677.sim +./test.sh -f issue/TD-2680.sim +./test.sh -f unique/dnode/lossdata.sim + +#======================b4-end=============== +#======================b5-start=============== + +./test.sh -f unique/dnode/alternativeRole.sim +./test.sh -f unique/dnode/balance1.sim +./test.sh -f unique/dnode/balance2.sim +./test.sh -f unique/dnode/balance3.sim +./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/offline1.sim +./test.sh -f unique/dnode/offline2.sim + +./test.sh -f general/stream/metrics_del.sim +./test.sh -f general/stream/metrics_replica1_vnoden.sim +./test.sh -f general/stream/restart_stream.sim +./test.sh -f general/stream/stream_3.sim +./test.sh -f general/stream/stream_restart.sim +./test.sh -f general/stream/table_del.sim +./test.sh -f general/stream/table_replica1_vnoden.sim + ./test.sh -f general/connection/test_old_data.sim ./test.sh -f unique/dnode/datatrans_3node.sim ./test.sh -f unique/dnode/datatrans_3node_2.sim +./test.sh -f general/db/alter_tables_d2.sim +./test.sh -f general/db/alter_tables_v1.sim +./test.sh -f general/db/alter_tables_v4.sim + +#======================b5-end=============== +#======================b6-start=============== + +./test.sh -f unique/dnode/reason.sim +./test.sh -f unique/dnode/remove1.sim +./test.sh -f unique/dnode/remove2.sim +./test.sh -f unique/dnode/vnode_clean.sim + +./test.sh -f unique/db/commit.sim +./test.sh -f unique/db/delete.sim +./test.sh -f unique/db/delete_part.sim +./test.sh -f unique/db/replica_add12.sim +./test.sh -f unique/db/replica_add13.sim +./test.sh -f unique/db/replica_add23.sim +./test.sh -f unique/db/replica_reduce21.sim +./test.sh -f unique/db/replica_reduce32.sim +./test.sh -f unique/db/replica_reduce31.sim +./test.sh -f unique/db/replica_part.sim + +./test.sh -f unique/vnode/many.sim +./test.sh -f unique/vnode/replica2_basic2.sim +./test.sh -f unique/vnode/replica2_repeat.sim +./test.sh -f unique/vnode/replica3_basic.sim +./test.sh -f unique/vnode/replica3_repeat.sim +./test.sh -f unique/vnode/replica3_vgroup.sim + +./test.sh -f unique/dnode/monitor.sim +./test.sh -f unique/dnode/monitor_bug.sim +./test.sh -f unique/dnode/simple.sim +./test.sh -f unique/dnode/data1.sim +./test.sh -f unique/dnode/m2.sim +./test.sh -f unique/dnode/m3.sim +./test.sh -f unique/dnode/offline3.sim +./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim + +./test.sh -f general/import/basic.sim +./test.sh -f general/import/commit.sim +./test.sh -f general/import/large.sim +./test.sh -f general/import/replica1.sim +./test.sh -f unique/cluster/balance1.sim +./test.sh -f unique/cluster/balance2.sim +./test.sh -f unique/cluster/balance3.sim + +#======================b6-end=============== +#======================b7-start=============== + +./test.sh -f general/compute/avg.sim +./test.sh -f general/compute/bottom.sim +./test.sh -f general/compute/count.sim +./test.sh -f general/compute/diff.sim +./test.sh -f general/compute/diff2.sim +./test.sh -f general/compute/first.sim +./test.sh -f general/compute/interval.sim +./test.sh -f general/compute/last.sim +./test.sh -f general/compute/leastsquare.sim +./test.sh -f general/compute/max.sim +./test.sh -f general/compute/min.sim +./test.sh -f general/compute/null.sim +./test.sh -f general/compute/percentile.sim +./test.sh -f general/compute/stddev.sim +./test.sh -f general/compute/sum.sim +./test.sh -f general/compute/top.sim + +./test.sh -f general/db/alter_option.sim +./test.sh -f general/db/alter_vgroups.sim +./test.sh -f general/db/basic.sim +./test.sh -f general/db/basic1.sim +./test.sh -f general/db/basic2.sim +./test.sh -f general/db/basic3.sim +./test.sh -f general/db/basic4.sim +./test.sh -f general/db/basic5.sim +./test.sh -f general/db/delete_reuse1.sim +./test.sh -f general/db/delete_reuse2.sim +./test.sh -f general/db/delete_reusevnode.sim +./test.sh -f general/db/delete_reusevnode2.sim +./test.sh -f general/db/delete_writing1.sim +./test.sh -f general/db/delete_writing2.sim +./test.sh -f general/db/delete.sim +./test.sh -f general/db/len.sim +./test.sh -f general/db/repeat.sim +./test.sh -f general/db/tables.sim +./test.sh -f general/db/vnodes.sim +./test.sh -f general/db/topic1.sim +./test.sh -f general/db/topic2.sim +./test.sh -f general/table/autocreate.sim +./test.sh -f general/table/basic1.sim +./test.sh -f general/table/basic2.sim +./test.sh -f general/table/basic3.sim +./test.sh -f general/table/bigint.sim +./test.sh -f general/table/binary.sim +./test.sh -f general/table/bool.sim +./test.sh -f general/table/column_name.sim +./test.sh -f general/table/column_num.sim +./test.sh -f general/table/column_value.sim +./test.sh -f general/table/column2.sim +./test.sh -f general/table/date.sim +./test.sh -f general/table/db.table.sim +./test.sh -f general/table/delete_reuse1.sim +./test.sh -f general/table/delete_reuse2.sim +./test.sh -f general/table/delete_writing.sim +./test.sh -f general/table/describe.sim +./test.sh -f general/table/double.sim +./test.sh -f general/table/fill.sim +./test.sh -f general/table/float.sim +./test.sh -f general/table/int.sim +./test.sh -f general/table/limit.sim +./test.sh -f general/table/smallint.sim +./test.sh -f general/table/table_len.sim +./test.sh -f general/table/table.sim +./test.sh -f general/table/tinyint.sim +./test.sh -f general/table/vgroup.sim +./test.sh -f general/table/createmulti.sim + +./test.sh -f unique/mnode/mgmt20.sim +./test.sh -f unique/mnode/mgmt21.sim +./test.sh -f unique/mnode/mgmt22.sim +./test.sh -f unique/mnode/mgmt23.sim +./test.sh -f unique/mnode/mgmt24.sim +./test.sh -f unique/mnode/mgmt25.sim +./test.sh -f unique/mnode/mgmt26.sim +./test.sh -f unique/mnode/mgmt33.sim +./test.sh -f unique/mnode/mgmt34.sim +./test.sh -f unique/mnode/mgmtr2.sim + +./test.sh -f unique/arbitrator/insert_duplicationTs.sim +./test.sh -f general/parser/join_manyblocks.sim +./test.sh -f general/parser/stableOp.sim +./test.sh -f general/parser/timestamp.sim +./test.sh -f general/parser/sliding.sim +./test.sh -f unique/big/balance.sim +#======================b7-end=============== diff --git a/tests/script/jenkins/basic_1.txt b/tests/script/jenkins/basic_1.txt deleted file mode 100644 index c124b60f806ff2e082289fa7e98b896a77e9d3b9..0000000000000000000000000000000000000000 --- a/tests/script/jenkins/basic_1.txt +++ /dev/null @@ -1,160 +0,0 @@ - -./test.sh -f general/compute/avg.sim -./test.sh -f general/compute/bottom.sim -./test.sh -f general/compute/count.sim -./test.sh -f general/compute/diff.sim -./test.sh -f general/compute/diff2.sim -./test.sh -f general/compute/first.sim -./test.sh -f general/compute/interval.sim -./test.sh -f general/compute/last.sim -./test.sh -f general/compute/leastsquare.sim -./test.sh -f general/compute/max.sim -./test.sh -f general/compute/min.sim -./test.sh -f general/compute/null.sim -./test.sh -f general/compute/percentile.sim -./test.sh -f general/compute/stddev.sim -./test.sh -f general/compute/sum.sim -./test.sh -f general/compute/top.sim - -./test.sh -f general/db/alter_option.sim -./test.sh -f general/db/alter_tables_d2.sim -./test.sh -f general/db/alter_tables_v1.sim -./test.sh -f general/db/alter_tables_v4.sim -./test.sh -f general/db/alter_vgroups.sim -./test.sh -f general/db/basic.sim -./test.sh -f general/db/basic1.sim -./test.sh -f general/db/basic2.sim -./test.sh -f general/db/basic3.sim -./test.sh -f general/db/basic4.sim -./test.sh -f general/db/basic5.sim -./test.sh -f general/db/delete_reuse1.sim -./test.sh -f general/db/delete_reuse2.sim -./test.sh -f general/db/delete_reusevnode.sim -./test.sh -f general/db/delete_reusevnode2.sim -./test.sh -f general/db/delete_writing1.sim -./test.sh -f general/db/delete_writing2.sim -./test.sh -f general/db/delete.sim -./test.sh -f general/db/len.sim -./test.sh -f general/db/repeat.sim -./test.sh -f general/db/tables.sim -./test.sh -f general/db/vnodes.sim - -./test.sh -f general/field/2.sim -./test.sh -f general/field/3.sim -./test.sh -f general/field/4.sim -./test.sh -f general/field/5.sim -./test.sh -f general/field/6.sim -./test.sh -f general/field/bigint.sim -./test.sh -f general/field/binary.sim -./test.sh -f general/field/bool.sim -./test.sh -f general/field/single.sim -./test.sh -f general/field/smallint.sim -./test.sh -f general/field/tinyint.sim - -./test.sh -f general/http/autocreate.sim -./test.sh -f general/http/chunked.sim -./test.sh -f general/http/gzip.sim -./test.sh -f general/http/restful.sim -./test.sh -f general/http/restful_insert.sim -./test.sh -f general/http/restful_limit.sim -./test.sh -f general/http/restful_full.sim -./test.sh -f general/http/prepare.sim -./test.sh -f general/http/telegraf.sim -./test.sh -f general/http/grafana_bug.sim -./test.sh -f general/http/grafana.sim - -./test.sh -f general/import/basic.sim -./test.sh -f general/import/commit.sim -./test.sh -f general/import/large.sim -./test.sh -f general/import/replica1.sim - -./test.sh -f general/insert/basic.sim -./test.sh -f general/insert/insert_drop.sim -./test.sh -f general/insert/query_block1_memory.sim -./test.sh -f general/insert/query_block2_memory.sim -./test.sh -f general/insert/query_block1_file.sim -./test.sh -f general/insert/query_block2_file.sim -./test.sh -f general/insert/query_file_memory.sim -./test.sh -f general/insert/query_multi_file.sim -./test.sh -f general/insert/tcp.sim - -./test.sh -f general/parser/alter.sim -./test.sh -f general/parser/alter1.sim -./test.sh -f general/parser/alter_stable.sim -./test.sh -f general/parser/auto_create_tb.sim -./test.sh -f general/parser/auto_create_tb_drop_tb.sim -./test.sh -f general/parser/col_arithmetic_operation.sim -./test.sh -f general/parser/columnValue.sim -./test.sh -f general/parser/commit.sim -./test.sh -f general/parser/create_db.sim -./test.sh -f general/parser/create_mt.sim -./test.sh -f general/parser/create_tb.sim -./test.sh -f general/parser/dbtbnameValidate.sim -./test.sh -f general/parser/import_commit1.sim -./test.sh -f general/parser/import_commit2.sim -./test.sh -f general/parser/import_commit3.sim -./test.sh -f general/parser/insert_tb.sim -./test.sh -f general/parser/first_last.sim -./test.sh -f general/parser/lastrow.sim -./test.sh -f general/parser/nchar.sim -./test.sh -f general/parser/null_char.sim -./test.sh -f general/parser/single_row_in_tb.sim -./test.sh -f general/parser/select_from_cache_disk.sim -./test.sh -f general/parser/mixed_blocks.sim -./test.sh -f general/parser/selectResNum.sim -./test.sh -f general/parser/limit.sim -./test.sh -f general/parser/limit1.sim -./test.sh -f general/parser/limit1_tblocks100.sim -./test.sh -f general/parser/select_across_vnodes.sim -./test.sh -f general/parser/slimit1.sim -./test.sh -f general/parser/tbnameIn.sim -./test.sh -f general/parser/projection_limit_offset.sim -./test.sh -f general/parser/limit2.sim -./test.sh -f general/parser/fill.sim -./test.sh -f general/parser/fill_stb.sim -./test.sh -f general/parser/where.sim -./test.sh -f general/parser/slimit.sim -./test.sh -f general/parser/select_with_tags.sim -./test.sh -f general/parser/interp.sim -./test.sh -f general/parser/tags_dynamically_specifiy.sim -./test.sh -f general/parser/groupby.sim -./test.sh -f general/parser/set_tag_vals.sim -./test.sh -f general/parser/tags_filter.sim -./test.sh -f general/parser/slimit_alter_tags.sim -./test.sh -f general/parser/join.sim -./test.sh -f general/parser/join_multivnode.sim -./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/repeatAlter.sim -./test.sh -f general/parser/union.sim -./test.sh -f general/parser/topbot.sim -./test.sh -f general/db/nosuchfile.sim -./test.sh -f general/parser/function.sim - -./test.sh -f general/table/autocreate.sim -./test.sh -f general/table/basic1.sim -./test.sh -f general/table/basic2.sim -./test.sh -f general/table/basic3.sim -./test.sh -f general/table/bigint.sim -./test.sh -f general/table/binary.sim -./test.sh -f general/table/bool.sim -./test.sh -f general/table/column_name.sim -./test.sh -f general/table/column_num.sim -./test.sh -f general/table/column_value.sim -./test.sh -f general/table/column2.sim -./test.sh -f general/table/date.sim -./test.sh -f general/table/db.table.sim -./test.sh -f general/table/delete_reuse1.sim -./test.sh -f general/table/delete_reuse2.sim -./test.sh -f general/table/delete_writing.sim -./test.sh -f general/table/describe.sim -./test.sh -f general/table/double.sim -./test.sh -f general/table/fill.sim -./test.sh -f general/table/float.sim -./test.sh -f general/table/int.sim -./test.sh -f general/table/limit.sim -./test.sh -f general/table/smallint.sim -./test.sh -f general/table/table_len.sim -./test.sh -f general/table/table.sim -./test.sh -f general/table/tinyint.sim -./test.sh -f general/table/vgroup.sim - diff --git a/tests/script/jenkins/basic_2.txt b/tests/script/jenkins/basic_2.txt deleted file mode 100644 index 4f5651219bf3fc8d87e9f95c73963ed16a2f28fd..0000000000000000000000000000000000000000 --- a/tests/script/jenkins/basic_2.txt +++ /dev/null @@ -1,104 +0,0 @@ -cd ../../../debug; cmake .. -cd ../../../debug; make - -./test.sh -f general/tag/3.sim -./test.sh -f general/tag/4.sim -./test.sh -f general/tag/5.sim -./test.sh -f general/tag/6.sim -./test.sh -f general/tag/add.sim -./test.sh -f general/tag/bigint.sim -./test.sh -f general/tag/binary_binary.sim -./test.sh -f general/tag/binary.sim -./test.sh -f general/tag/bool_binary.sim -./test.sh -f general/tag/bool_int.sim -./test.sh -f general/tag/bool.sim -./test.sh -f general/tag/change.sim -./test.sh -f general/tag/column.sim -./test.sh -f general/tag/commit.sim -./test.sh -f general/tag/create.sim -./test.sh -f general/tag/delete.sim -./test.sh -f general/tag/double.sim -./test.sh -f general/tag/filter.sim -./test.sh -f general/tag/float.sim -./test.sh -f general/tag/int_binary.sim -./test.sh -f general/tag/int_float.sim -./test.sh -f general/tag/int.sim -./test.sh -f general/tag/set.sim -./test.sh -f general/tag/smallint.sim -./test.sh -f general/tag/tinyint.sim -./test.sh -f general/wal/sync.sim -./test.sh -f general/wal/kill.sim -./test.sh -f general/wal/maxtables.sim - -./test.sh -f general/user/authority.sim -./test.sh -f general/user/monitor.sim -./test.sh -f general/user/pass_alter.sim -./test.sh -f general/user/pass_len.sim -./test.sh -f general/user/user_create.sim -./test.sh -f general/user/user_len.sim - -./test.sh -f general/vector/metrics_field.sim -./test.sh -f general/vector/metrics_mix.sim -./test.sh -f general/vector/metrics_query.sim -./test.sh -f general/vector/metrics_tag.sim -./test.sh -f general/vector/metrics_time.sim -./test.sh -f general/vector/multi.sim -./test.sh -f general/vector/single.sim -./test.sh -f general/vector/table_field.sim -./test.sh -f general/vector/table_mix.sim -./test.sh -f general/vector/table_query.sim -./test.sh -f general/vector/table_time.sim - -./test.sh -f unique/account/account_create.sim -./test.sh -f unique/account/account_delete.sim -./test.sh -f unique/account/account_len.sim -./test.sh -f unique/account/authority.sim -./test.sh -f unique/account/basic.sim -./test.sh -f unique/account/paras.sim -./test.sh -f unique/account/pass_alter.sim -./test.sh -f unique/account/pass_len.sim -./test.sh -f unique/account/usage.sim -./test.sh -f unique/account/user_create.sim -./test.sh -f unique/account/user_len.sim - -./test.sh -f unique/big/balance.sim -./test.sh -f unique/big/maxvnodes.sim -./test.sh -f unique/big/tcp.sim - -./test.sh -f unique/cluster/alter.sim -./test.sh -f unique/cluster/balance1.sim -./test.sh -f unique/cluster/balance2.sim -./test.sh -f unique/cluster/balance3.sim -./test.sh -f unique/cluster/cache.sim -./test.sh -f unique/cluster/vgroup100.sim - -./test.sh -f unique/column/replica3.sim - -./test.sh -f unique/db/commit.sim -./test.sh -f unique/db/delete.sim -./test.sh -f unique/db/delete_part.sim -./test.sh -f unique/db/replica_add12.sim -./test.sh -f unique/db/replica_add13.sim -./test.sh -f unique/db/replica_add23.sim -./test.sh -f unique/db/replica_reduce21.sim -./test.sh -f unique/db/replica_reduce32.sim -./test.sh -f unique/db/replica_reduce31.sim -./test.sh -f unique/db/replica_part.sim - -./test.sh -f unique/vnode/many.sim -./test.sh -f unique/vnode/replica2_basic2.sim -./test.sh -f unique/vnode/replica2_repeat.sim -./test.sh -f unique/vnode/replica3_basic.sim -./test.sh -f unique/vnode/replica3_repeat.sim -./test.sh -f unique/vnode/replica3_vgroup.sim - -./test.sh -f unique/dnode/monitor.sim -./test.sh -f unique/dnode/monitor_bug.sim -./test.sh -f unique/dnode/simple.sim -./test.sh -f unique/dnode/data1.sim -./test.sh -f unique/dnode/m2.sim -./test.sh -f unique/dnode/m3.sim -./test.sh -f unique/dnode/offline3.sim -./test.sh -f general/wal/sync.sim -./test.sh -f general/wal/kill.sim -./test.sh -f general/wal/maxtables.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_3.txt b/tests/script/jenkins/basic_3.txt deleted file mode 100644 index 25bfde28f0b724335e248933f01e72c3c35fd3dc..0000000000000000000000000000000000000000 --- a/tests/script/jenkins/basic_3.txt +++ /dev/null @@ -1,79 +0,0 @@ - -./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim -#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim -# lower the priority while file corruption -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim -#./test.sh -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim -./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim -./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim -./test.sh -f unique/arbitrator/insert_duplicationTs.sim -./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim -./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim -./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim -./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim - -./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim -./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim -./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim -./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim - -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim -./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim - -./test.sh -f unique/stable/balance_replica1.sim -./test.sh -f unique/stable/dnode2_stop.sim -./test.sh -f unique/stable/dnode2.sim -./test.sh -f unique/stable/dnode3.sim -./test.sh -f unique/stable/replica2_dnode4.sim -./test.sh -f unique/stable/replica2_vnode3.sim -./test.sh -f unique/stable/replica3_dnode6.sim -./test.sh -f unique/stable/replica3_vnode3.sim - -./test.sh -f unique/mnode/mgmt20.sim -./test.sh -f unique/mnode/mgmt21.sim -./test.sh -f unique/mnode/mgmt22.sim -./test.sh -f unique/mnode/mgmt23.sim -./test.sh -f unique/mnode/mgmt24.sim -./test.sh -f unique/mnode/mgmt25.sim -./test.sh -f unique/mnode/mgmt26.sim -./test.sh -f unique/mnode/mgmt33.sim -./test.sh -f unique/mnode/mgmt34.sim -./test.sh -f unique/mnode/mgmtr2.sim - -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - -./test.sh -f general/connection/test_old_data.sim -./test.sh -f unique/dnode/datatrans_3node.sim -./test.sh -f unique/dnode/datatrans_3node_2.sim \ No newline at end of file diff --git a/tests/script/jenkins/basic_4.txt b/tests/script/jenkins/basic_4.txt deleted file mode 100644 index 895281f218717af7dac3b8e1f890ce3e03b81609..0000000000000000000000000000000000000000 --- a/tests/script/jenkins/basic_4.txt +++ /dev/null @@ -1,49 +0,0 @@ -./test.sh -f unique/dnode/alternativeRole.sim -./test.sh -f unique/dnode/balance1.sim -./test.sh -f unique/dnode/balance2.sim -./test.sh -f unique/dnode/balance3.sim -./test.sh -f unique/dnode/balancex.sim -./test.sh -f unique/dnode/offline1.sim -./test.sh -f unique/dnode/offline2.sim -./test.sh -f unique/dnode/reason.sim -./test.sh -f unique/dnode/remove1.sim -./test.sh -f unique/dnode/remove2.sim -./test.sh -f unique/dnode/vnode_clean.sim - -./test.sh -f unique/http/admin.sim -./test.sh -f unique/http/opentsdb.sim - -./test.sh -f unique/import/replica2.sim -./test.sh -f unique/import/replica3.sim - -./test.sh -f general/alter/cached_schema_after_alter.sim -./test.sh -f general/alter/count.sim -./test.sh -f general/alter/dnode.sim -./test.sh -f general/alter/import.sim -./test.sh -f general/alter/insert1.sim -./test.sh -f general/alter/insert2.sim -./test.sh -f general/alter/metrics.sim -./test.sh -f general/alter/table.sim - -./test.sh -f general/cache/new_metrics.sim -./test.sh -f general/cache/restart_metrics.sim -./test.sh -f general/cache/restart_table.sim - -./test.sh -f general/connection/connection.sim - -./test.sh -f general/column/commit.sim -./test.sh -f general/column/metrics.sim -./test.sh -f general/column/table.sim - -./test.sh -f general/compress/commitlog.sim -./test.sh -f general/compress/compress.sim -./test.sh -f general/compress/compress2.sim -./test.sh -f general/compress/uncompress.sim - -./test.sh -f general/stable/disk.sim -./test.sh -f general/stable/dnode3.sim -./test.sh -f general/stable/metrics.sim -./test.sh -f general/stable/refcount.sim -./test.sh -f general/stable/show.sim -./test.sh -f general/stable/values.sim -./test.sh -f general/stable/vnode3.sim \ No newline at end of file diff --git a/tests/script/jenkins/wbasic.txt b/tests/script/jenkins/wbasic.txt new file mode 100644 index 0000000000000000000000000000000000000000..984b15fc5a4e49a5e730667782b41afb8bdb6bb8 --- /dev/null +++ b/tests/script/jenkins/wbasic.txt @@ -0,0 +1,393 @@ +wtest.bat -f issue/TD-2677.sim +wtest.bat -f issue/TD-2680.sim +wtest.bat -f issue/TD-2713.sim + +wtest.bat -f general/alter/cached_schema_after_alter.sim +wtest.bat -f general/alter/count.sim +wtest.bat -f general/alter/dnode.sim +wtest.bat -f general/alter/import.sim +wtest.bat -f general/alter/insert1.sim +wtest.bat -f general/alter/insert2.sim +wtest.bat -f general/alter/metrics.sim +wtest.bat -f general/alter/table.sim + +wtest.bat -f general/cache/new_metrics.sim +wtest.bat -f general/cache/restart_metrics.sim +wtest.bat -f general/cache/restart_table.sim + +wtest.bat -f general/connection/connection.sim + +wtest.bat -f general/column/commit.sim +wtest.bat -f general/column/metrics.sim +wtest.bat -f general/column/table.sim + +wtest.bat -f general/compress/commitlog.sim +wtest.bat -f general/compress/compress.sim +wtest.bat -f general/compress/compress2.sim +wtest.bat -f general/compress/uncompress.sim + +wtest.bat -f general/compute/avg.sim +wtest.bat -f general/compute/bottom.sim +wtest.bat -f general/compute/count.sim +wtest.bat -f general/compute/diff.sim +wtest.bat -f general/compute/diff2.sim +wtest.bat -f general/compute/first.sim +wtest.bat -f general/compute/interval.sim +wtest.bat -f general/compute/last.sim +wtest.bat -f general/compute/leastsquare.sim +wtest.bat -f general/compute/max.sim +wtest.bat -f general/compute/min.sim +wtest.bat -f general/compute/null.sim +wtest.bat -f general/compute/percentile.sim +wtest.bat -f general/compute/stddev.sim +wtest.bat -f general/compute/sum.sim +wtest.bat -f general/compute/top.sim + +wtest.bat -f general/db/alter_option.sim +wtest.bat -f general/db/alter_tables_d2.sim +wtest.bat -f general/db/alter_tables_v1.sim +wtest.bat -f general/db/alter_tables_v4.sim +wtest.bat -f general/db/alter_vgroups.sim +wtest.bat -f general/db/basic.sim +wtest.bat -f general/db/basic1.sim +wtest.bat -f general/db/basic2.sim +wtest.bat -f general/db/basic3.sim +wtest.bat -f general/db/basic4.sim +wtest.bat -f general/db/basic5.sim +wtest.bat -f general/db/delete_reuse1.sim +wtest.bat -f general/db/delete_reuse2.sim +wtest.bat -f general/db/delete_reusevnode.sim +wtest.bat -f general/db/delete_reusevnode2.sim +wtest.bat -f general/db/delete_writing1.sim +wtest.bat -f general/db/delete_writing2.sim +wtest.bat -f general/db/delete.sim +wtest.bat -f general/db/len.sim +wtest.bat -f general/db/repeat.sim +wtest.bat -f general/db/tables.sim +wtest.bat -f general/db/vnodes.sim +wtest.bat -f general/db/nosuchfile.sim + +wtest.bat -f general/field/2.sim +wtest.bat -f general/field/3.sim +wtest.bat -f general/field/4.sim +wtest.bat -f general/field/5.sim +wtest.bat -f general/field/6.sim +wtest.bat -f general/field/bigint.sim +wtest.bat -f general/field/binary.sim +wtest.bat -f general/field/bool.sim +wtest.bat -f general/field/single.sim +wtest.bat -f general/field/smallint.sim +wtest.bat -f general/field/tinyint.sim + +wtest.bat -f general/http/autocreate.sim +wtest.bat -f general/http/chunked.sim +wtest.bat -f general/http/gzip.sim +wtest.bat -f general/http/restful.sim +wtest.bat -f general/http/restful_insert.sim +wtest.bat -f general/http/restful_limit.sim +wtest.bat -f general/http/restful_full.sim +wtest.bat -f general/http/prepare.sim +wtest.bat -f general/http/telegraf.sim +wtest.bat -f general/http/grafana_bug.sim +wtest.bat -f general/http/grafana.sim + +wtest.bat -f general/import/basic.sim +wtest.bat -f general/import/commit.sim +wtest.bat -f general/import/large.sim +wtest.bat -f general/import/replica1.sim + +wtest.bat -f general/insert/basic.sim +wtest.bat -f general/insert/insert_drop.sim +wtest.bat -f general/insert/query_block1_memory.sim +wtest.bat -f general/insert/query_block2_memory.sim +wtest.bat -f general/insert/query_block1_file.sim +wtest.bat -f general/insert/query_block2_file.sim +wtest.bat -f general/insert/query_file_memory.sim +wtest.bat -f general/insert/query_multi_file.sim +wtest.bat -f general/insert/tcp.sim + +wtest.bat -f general/parser/alter.sim +wtest.bat -f general/parser/alter1.sim +wtest.bat -f general/parser/alter_stable.sim +wtest.bat -f general/parser/auto_create_tb.sim +wtest.bat -f general/parser/auto_create_tb_drop_tb.sim +wtest.bat -f general/parser/col_arithmetic_operation.sim +wtest.bat -f general/parser/columnValue.sim +wtest.bat -f general/parser/commit.sim +wtest.bat -f general/parser/create_db.sim +wtest.bat -f general/parser/create_mt.sim +wtest.bat -f general/parser/create_tb.sim +wtest.bat -f general/parser/dbtbnameValidate.sim +wtest.bat -f general/parser/import_commit1.sim +wtest.bat -f general/parser/import_commit2.sim +wtest.bat -f general/parser/import_commit3.sim +wtest.bat -f general/parser/insert_tb.sim +wtest.bat -f general/parser/first_last.sim +wtest.bat -f general/parser/lastrow.sim +wtest.bat -f general/parser/nchar.sim +wtest.bat -f general/parser/null_char.sim +wtest.bat -f general/parser/single_row_in_tb.sim +wtest.bat -f general/parser/select_from_cache_disk.sim +wtest.bat -f general/parser/mixed_blocks.sim +wtest.bat -f general/parser/selectResNum.sim +wtest.bat -f general/parser/limit.sim +wtest.bat -f general/parser/limit1.sim +wtest.bat -f general/parser/limit1_tblocks100.sim +wtest.bat -f general/parser/select_across_vnodes.sim +wtest.bat -f general/parser/slimit1.sim +wtest.bat -f general/parser/tbnameIn.sim +wtest.bat -f general/parser/projection_limit_offset.sim +wtest.bat -f general/parser/limit2.sim +wtest.bat -f general/parser/fill.sim +wtest.bat -f general/parser/fill_stb.sim +wtest.bat -f general/parser/where.sim +wtest.bat -f general/parser/slimit.sim +wtest.bat -f general/parser/select_with_tags.sim +wtest.bat -f general/parser/interp.sim +wtest.bat -f general/parser/tags_dynamically_specifiy.sim +wtest.bat -f general/parser/groupby.sim +wtest.bat -f general/parser/set_tag_vals.sim +wtest.bat -f general/parser/tags_filter.sim +wtest.bat -f general/parser/slimit_alter_tags.sim +wtest.bat -f general/parser/join.sim +wtest.bat -f general/parser/join_multivnode.sim +wtest.bat -f general/parser/binary_escapeCharacter.sim +wtest.bat -f general/parser/repeatAlter.sim +wtest.bat -f general/parser/union.sim +wtest.bat -f general/parser/topbot.sim +wtest.bat -f general/parser/function.sim + +wtest.bat -f general/stable/disk.sim +wtest.bat -f general/stable/dnode3.sim +wtest.bat -f general/stable/metrics.sim +wtest.bat -f general/stable/refcount.sim +wtest.bat -f general/stable/show.sim +wtest.bat -f general/stable/values.sim +wtest.bat -f general/stable/vnode3.sim + +wtest.bat -f general/table/autocreate.sim +wtest.bat -f general/table/basic1.sim +wtest.bat -f general/table/basic2.sim +wtest.bat -f general/table/basic3.sim +wtest.bat -f general/table/bigint.sim +wtest.bat -f general/table/binary.sim +wtest.bat -f general/table/bool.sim +wtest.bat -f general/table/column_name.sim +wtest.bat -f general/table/column_num.sim +wtest.bat -f general/table/column_value.sim +wtest.bat -f general/table/column2.sim +wtest.bat -f general/table/date.sim +wtest.bat -f general/table/db.table.sim +wtest.bat -f general/table/delete_reuse1.sim +wtest.bat -f general/table/delete_reuse2.sim +wtest.bat -f general/table/delete_writing.sim +wtest.bat -f general/table/describe.sim +wtest.bat -f general/table/double.sim +wtest.bat -f general/table/fill.sim +wtest.bat -f general/table/float.sim +wtest.bat -f general/table/int.sim +wtest.bat -f general/table/limit.sim +wtest.bat -f general/table/smallint.sim +wtest.bat -f general/table/table_len.sim +wtest.bat -f general/table/table.sim +wtest.bat -f general/table/tinyint.sim +wtest.bat -f general/table/vgroup.sim + +wtest.bat -f general/tag/3.sim +wtest.bat -f general/tag/4.sim +wtest.bat -f general/tag/5.sim +wtest.bat -f general/tag/6.sim +wtest.bat -f general/tag/add.sim +wtest.bat -f general/tag/bigint.sim +wtest.bat -f general/tag/binary_binary.sim +wtest.bat -f general/tag/binary.sim +wtest.bat -f general/tag/bool_binary.sim +wtest.bat -f general/tag/bool_int.sim +wtest.bat -f general/tag/bool.sim +wtest.bat -f general/tag/change.sim +wtest.bat -f general/tag/column.sim +wtest.bat -f general/tag/commit.sim +wtest.bat -f general/tag/create.sim +wtest.bat -f general/tag/delete.sim +wtest.bat -f general/tag/double.sim +wtest.bat -f general/tag/filter.sim +wtest.bat -f general/tag/float.sim +wtest.bat -f general/tag/int_binary.sim +wtest.bat -f general/tag/int_float.sim +wtest.bat -f general/tag/int.sim +wtest.bat -f general/tag/set.sim +wtest.bat -f general/tag/smallint.sim +wtest.bat -f general/tag/tinyint.sim + +wtest.bat -f general/user/authority.sim +wtest.bat -f general/user/monitor.sim +wtest.bat -f general/user/pass_alter.sim +wtest.bat -f general/user/pass_len.sim +wtest.bat -f general/user/user_create.sim +wtest.bat -f general/user/user_len.sim + +wtest.bat -f general/vector/metrics_field.sim +wtest.bat -f general/vector/metrics_mix.sim +wtest.bat -f general/vector/metrics_query.sim +wtest.bat -f general/vector/metrics_tag.sim +wtest.bat -f general/vector/metrics_time.sim +wtest.bat -f general/vector/multi.sim +wtest.bat -f general/vector/single.sim +wtest.bat -f general/vector/table_field.sim +wtest.bat -f general/vector/table_mix.sim +wtest.bat -f general/vector/table_query.sim +wtest.bat -f general/vector/table_time.sim + +wtest.bat -f general/wal/sync.sim +wtest.bat -f general/wal/kill.sim +wtest.bat -f general/wal/maxtables.sim + +wtest.bat -f unique/account/account_create.sim +wtest.bat -f unique/account/account_delete.sim +wtest.bat -f unique/account/account_len.sim +wtest.bat -f unique/account/authority.sim +wtest.bat -f unique/account/basic.sim +wtest.bat -f unique/account/paras.sim +wtest.bat -f unique/account/pass_alter.sim +wtest.bat -f unique/account/pass_len.sim +wtest.bat -f unique/account/usage.sim +wtest.bat -f unique/account/user_create.sim +wtest.bat -f unique/account/user_len.sim + +wtest.bat -f unique/big/balance.sim +wtest.bat -f unique/big/maxvnodes.sim +wtest.bat -f unique/big/tcp.sim + +wtest.bat -f unique/cluster/alter.sim +wtest.bat -f unique/cluster/balance1.sim +wtest.bat -f unique/cluster/balance2.sim +wtest.bat -f unique/cluster/balance3.sim +wtest.bat -f unique/cluster/cache.sim +wtest.bat -f unique/cluster/vgroup100.sim + +wtest.bat -f unique/column/replica3.sim + +wtest.bat -f unique/db/commit.sim +wtest.bat -f unique/db/delete.sim +wtest.bat -f unique/db/delete_part.sim +wtest.bat -f unique/db/replica_add12.sim +wtest.bat -f unique/db/replica_add13.sim +wtest.bat -f unique/db/replica_add23.sim +wtest.bat -f unique/db/replica_reduce21.sim +wtest.bat -f unique/db/replica_reduce32.sim +wtest.bat -f unique/db/replica_reduce31.sim +wtest.bat -f unique/db/replica_part.sim + +wtest.bat -f unique/dnode/alternativeRole.sim +wtest.bat -f unique/dnode/monitor.sim +wtest.bat -f unique/dnode/monitor_bug.sim +wtest.bat -f unique/dnode/simple.sim +wtest.bat -f unique/dnode/balance1.sim +wtest.bat -f unique/dnode/balance2.sim +wtest.bat -f unique/dnode/balance3.sim +wtest.bat -f unique/dnode/balancex.sim +wtest.bat -f unique/dnode/data1.sim +wtest.bat -f unique/dnode/m2.sim +wtest.bat -f unique/dnode/m3.sim +wtest.bat -f unique/dnode/lossdata.sim +wtest.bat -f unique/dnode/offline1.sim +wtest.bat -f unique/dnode/offline2.sim +wtest.bat -f unique/dnode/offline3.sim +wtest.bat -f unique/dnode/reason.sim +wtest.bat -f unique/dnode/remove1.sim +wtest.bat -f unique/dnode/remove2.sim +wtest.bat -f unique/dnode/vnode_clean.sim + +wtest.bat -f unique/http/admin.sim +wtest.bat -f unique/http/opentsdb.sim + +wtest.bat -f unique/import/replica2.sim +wtest.bat -f unique/import/replica3.sim + +wtest.bat -f unique/stable/balance_replica1.sim +wtest.bat -f unique/stable/dnode2_stop.sim +wtest.bat -f unique/stable/dnode2.sim +wtest.bat -f unique/stable/dnode3.sim +wtest.bat -f unique/stable/replica2_dnode4.sim +wtest.bat -f unique/stable/replica2_vnode3.sim +wtest.bat -f unique/stable/replica3_dnode6.sim +wtest.bat -f unique/stable/replica3_vnode3.sim + +wtest.bat -f unique/mnode/mgmt20.sim +wtest.bat -f unique/mnode/mgmt21.sim +wtest.bat -f unique/mnode/mgmt22.sim +wtest.bat -f unique/mnode/mgmt23.sim +wtest.bat -f unique/mnode/mgmt24.sim +wtest.bat -f unique/mnode/mgmt25.sim +wtest.bat -f unique/mnode/mgmt26.sim +wtest.bat -f unique/mnode/mgmt30.sim +wtest.bat -f unique/mnode/mgmt33.sim +wtest.bat -f unique/mnode/mgmt34.sim +wtest.bat -f unique/mnode/mgmtr2.sim + +wtest.bat -f unique/vnode/many.sim +wtest.bat -f unique/vnode/replica2_basic2.sim +wtest.bat -f unique/vnode/replica2_repeat.sim +wtest.bat -f unique/vnode/replica3_basic.sim +wtest.bat -f unique/vnode/replica3_repeat.sim +wtest.bat -f unique/vnode/replica3_vgroup.sim + +wtest.bat -f general/stream/metrics_del.sim +wtest.bat -f general/stream/metrics_replica1_vnoden.sim +wtest.bat -f general/stream/restart_stream.sim +wtest.bat -f general/stream/stream_3.sim +wtest.bat -f general/stream/stream_restart.sim +wtest.bat -f general/stream/table_del.sim +wtest.bat -f general/stream/table_replica1_vnoden.sim + +wtest.bat -f unique/arbitrator/check_cluster_cfg_para.sim +#wtest.bat -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +wtest.bat -f unique/arbitrator/dn3_mn1_full_createTableFail.sim +wtest.bat -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +wtest.bat -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +wtest.bat -f unique/arbitrator/dn3_mn1_replica_change.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +# lower the priority while file corruption +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_change.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +wtest.bat -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim +wtest.bat -f unique/arbitrator/dn3_mn2_killDnode.sim +wtest.bat -f unique/arbitrator/insert_duplicationTs.sim +wtest.bat -f unique/arbitrator/offline_replica2_alterTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_alterTag_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_createTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_dropDb_online.sim +wtest.bat -f unique/arbitrator/offline_replica2_dropTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_alterTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_alterTag_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_createTable_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_dropDb_online.sim +wtest.bat -f unique/arbitrator/offline_replica3_dropTable_online.sim +wtest.bat -f unique/arbitrator/replica_changeWithArbitrator.sim +wtest.bat -f unique/arbitrator/sync_replica2_alterTable_add.sim +wtest.bat -f unique/arbitrator/sync_replica2_alterTable_drop.sim + +wtest.bat -f unique/arbitrator/sync_replica2_dropDb.sim +wtest.bat -f unique/arbitrator/sync_replica2_dropTable.sim +wtest.bat -f unique/arbitrator/sync_replica3_alterTable_add.sim +wtest.bat -f unique/arbitrator/sync_replica3_alterTable_drop.sim +wtest.bat -f unique/arbitrator/sync_replica3_dropDb.sim +wtest.bat -f unique/arbitrator/sync_replica3_dropTable.sim + +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +wtest.bat -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim + +wtest.bat -f general/connection/test_old_data.sim +wtest.bat -f unique/dnode/datatrans_3node.sim +wtest.bat -f unique/dnode/datatrans_3node_2.sim + diff --git a/tests/script/sh/cfg.bat b/tests/script/sh/cfg.bat new file mode 100644 index 0000000000000000000000000000000000000000..49ab34383a6a6392b40314c912dd2931dbaf7a36 --- /dev/null +++ b/tests/script/sh/cfg.bat @@ -0,0 +1,42 @@ +@echo off + +if %1 == -n set NODE_NAME=%2 +if %1 == -c set CONFIG_NAME=%2 +if %1 == -v set CONFIG_VALUE=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -c set CONFIG_NAME=%4 +if %3 == -v set CONFIG_VALUE=%4 +if %5 == -n set NODE_NAME=%6 +if %5 == -c set CONFIG_NAME=%6 +if %5 == -v set CONFIG_VALUE=%6 + +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %NODE% + +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% + +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TSIM=%BUILD_DIR%tsim +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% + +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% + +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% + +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% + +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% + +set DATA_DIR=%NODE_DIR%data\ +rem echo DATA_DIR: %DATA_DIR% + +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% + +echo %CONFIG_NAME% %CONFIG_VALUE% >> %TAOS_CFG% diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat new file mode 100644 index 0000000000000000000000000000000000000000..921f1611fb5ea80bbeb746b693d7c529c421ef27 --- /dev/null +++ b/tests/script/sh/deploy.bat @@ -0,0 +1,106 @@ +@echo off + +rem echo Executing deploy.sh + +if %1 == -n set NODE_NAME=%2 +if %1 == -i set NODE=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -i set NODE=%4 + +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %NODE% + +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% + +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TSIM=%BUILD_DIR%tsim +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% + +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% + +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% + +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% + +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% + +set DATA_DIR=%NODE_DIR%data\ +rem echo DATA_DIR: %DATA_DIR% + +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% + +if not exist %SIM_DIR% mkdir %SIM_DIR% +if not exist %NODE_DIR% mkdir %NODE_DIR% +if exist %CFG_DIR% rmdir /s/q %CFG_DIR% +if exist %LOG_DIR% rmdir /s/q %LOG_DIR% +if exist %DATA_DIR% rmdir /s/q %DATA_DIR% +if not exist %CFG_DIR% mkdir %CFG_DIR% +if not exist %LOG_DIR% mkdir %LOG_DIR% +if not exist %DATA_DIR% mkdir %DATA_DIR% + +if %NODE% == 1 set NODE=7100 +if %NODE% == 2 set NODE=7200 +if %NODE% == 3 set NODE=7300 +if %NODE% == 4 set NODE=7400 +if %NODE% == 5 set NODE=7500 +if %NODE% == 6 set NODE=7600 +if %NODE% == 7 set NODE=7700 +if %NODE% == 8 set NODE=7800 + +rem set "fqdn=" +for /f "skip=1" %%A in ( + 'wmic computersystem get caption' +) do if not defined fqdn set "fqdn=%%A" + +echo firstEp %fqdn%:7100 > %TAOS_CFG% +echo fqdn %fqdn% >> %TAOS_CFG% +echo serverPort %NODE% >> %TAOS_CFG% +echo dataDir %DATA_DIR% >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo debugFlag 0 >> %TAOS_CFG% +echo mDebugFlag 135 >> %TAOS_CFG% +echo sdbDebugFlag 135 >> %TAOS_CFG% +echo dDebugFlag 135 >> %TAOS_CFG% +echo vDebugFlag 135 >> %TAOS_CFG% +echo tsdbDebugFlag 135 >> %TAOS_CFG% +echo cDebugFlag 135 >> %TAOS_CFG% +echo jnidebugFlag 135 >> %TAOS_CFG% +echo odbcdebugFlag 135 >> %TAOS_CFG% +echo httpDebugFlag 135 >> %TAOS_CFG% +echo monDebugFlag 135 >> %TAOS_CFG% +echo mqttDebugFlag 135 >> %TAOS_CFG% +echo qdebugFlag 135 >> %TAOS_CFG% +echo rpcDebugFlag 135 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo udebugFlag 135 >> %TAOS_CFG% +echo sdebugFlag 135 >> %TAOS_CFG% +echo wdebugFlag 135 >> %TAOS_CFG% +echo cqdebugFlag 135 >> %TAOS_CFG% +echo monitor 0 >> %TAOS_CFG% +echo monitorInterval 1 >> %TAOS_CFG% +echo http 0 >> %TAOS_CFG% +echo slaveQuery 0 >> %TAOS_CFG% +echo numOfThreadsPerCore 2.0 >> %TAOS_CFG% +echo defaultPass taosdata >> %TAOS_CFG% +echo numOfLogLines 20000000 >> %TAOS_CFG% +echo mnodeEqualVnodeNum 0 >> %TAOS_CFG% +echo balanceInterval 1 >> %TAOS_CFG% +echo clog 2 >> %TAOS_CFG% +echo days 10 >> %TAOS_CFG% +echo statusInterval 1 >> %TAOS_CFG% +echo maxVgroupsPerDb 4 >> %TAOS_CFG% +echo minTablesPerVnode 4 >> %TAOS_CFG% +echo maxTablesPerVnode 1000 >> %TAOS_CFG% +echo tableIncStepPerVnode 10000 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo numOfMnodes 1 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo fsync 0 >> %TAOS_CFG% +echo telemetryReporting 0 >> %TAOS_CFG% diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat new file mode 100644 index 0000000000000000000000000000000000000000..6651c7aa8f726d5d42d52a4c8f3a9395e7a12151 --- /dev/null +++ b/tests/script/sh/exec.bat @@ -0,0 +1,48 @@ +@echo off + +rem echo Executing exec.sh + +if %1 == -n set NODE_NAME=%2 +if %1 == -s set EXEC_OPTON=%2 +if %3 == -n set NODE_NAME=%4 +if %3 == -s set EXEC_OPTON=%4 + +rem echo NODE_NAME: %NODE_NAME% +rem echo NODE: %EXEC_OPTON% + +set SCRIPT_DIR=%~dp0..\ +rem echo SCRIPT_DIR: %SCRIPT_DIR% + +set BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\ +set TAOSD=%BUILD_DIR%taosd +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TAOSD: %TAOSD% + +set SIM_DIR=%SCRIPT_DIR%..\..\..\sim\ +rem echo SIM_DIR: %SIM_DIR% + +set NODE_DIR=%SIM_DIR%%NODE_NAME%\ +rem echo NODE_DIR: %NODE_DIR% + +set CFG_DIR=%NODE_DIR%cfg\ +rem echo CFG_DIR: %CFG_DIR% + +set TAOS_CFG=%CFG_DIR%taos.cfg +rem echo TAOS_CFG: %TAOS_CFG% + +if %EXEC_OPTON% == start ( + echo start %TAOSD% -c %CFG_DIR% + start %TAOSD% -c %CFG_DIR% +) + +if %EXEC_OPTON% == stop ( + rem echo wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" list INSTANCE + rem wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" call terminate > NUL 2>&1 + + for /f "tokens=1 skip=1" %%A in ( + 'wmic process where "name='taosd.exe' and CommandLine like '%%%NODE_NAME%%%'" get processId ' + ) do ( + rem echo taskkill /IM %%A + taskkill /IM %%A > NUL 2>&1 + ) +) diff --git a/tests/script/sh/mv_old_data.sh b/tests/script/sh/mv_old_data.sh index 112e9760d637d9698b87729c06ba7b2bde9651f8..606ee171f1e6b8605fd19d542e0b0dce5dfdda28 100755 --- a/tests/script/sh/mv_old_data.sh +++ b/tests/script/sh/mv_old_data.sh @@ -32,6 +32,9 @@ NODE_DIR=$SIM_DIR/$NODE_NAME rm -rf $SIM_DIR/dnode1 rm -rf $SIM_DIR/dnode2 rm -rf $SIM_DIR/dnode3 -rm -rf $SIM_DIR/tsim tar zxf $SCRIPT_DIR/general/connection/sim.tar.gz -C $SIM_DIR/../ +cd $SIM_DIR/../sim +fqdn=`hostname -f || hostname` +grep 'test4' -l -r ./* | xargs sed -i "s/test4/${fqdn}/g" +grep 'dataDir' -l -r ./* | xargs sed -i "s#/root/TDengine#${TAOS_DIR}#g" diff --git a/tests/script/sh/stop_dnodes.bat b/tests/script/sh/stop_dnodes.bat new file mode 100644 index 0000000000000000000000000000000000000000..ab7af2ca92023745b8b712cb78e9b168a6c00598 --- /dev/null +++ b/tests/script/sh/stop_dnodes.bat @@ -0,0 +1,6 @@ +@echo off + +rem echo taskkill /F /IM taosd.exe + +wmic process where "name='taosd.exe'" call terminate > NUL 2>&1 +taskkill /F /IM taosd.exe > NUL 2>&1 \ No newline at end of file diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim index 23f59d1d00c296d5fc9d3724ce428375a419938e..8bca76c38b627ac6a6aa9eaae07a9126a9b19057 100644 --- a/tests/script/tmp/mnodes.sim +++ b/tests/script/tmp/mnodes.sim @@ -107,7 +107,7 @@ if $data4_3 != ready then endi $x = $x + 1 -sleep 5000 +sleep 3000 if $x == 100000 then return -1 endi diff --git a/tests/script/unique/account/account_create.sim b/tests/script/unique/account/account_create.sim index 9eecceb39d106ab335b631bcfae370c2ae516490..e36de29e7c5835ddc78a9f3eab4b2b4d34634c42 100644 --- a/tests/script/unique/account/account_create.sim +++ b/tests/script/unique/account/account_create.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start diff --git a/tests/script/unique/account/account_delete.sim b/tests/script/unique/account/account_delete.sim index 564512228eda3aa156faba4818e3bc150b69de76..d99a8b559dc6e04e4d6996e042d915671781d699 100644 --- a/tests/script/unique/account/account_delete.sim +++ b/tests/script/unique/account/account_delete.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/unique/account/account_len.sim b/tests/script/unique/account/account_len.sim index 75636be513af84ba00d9052d09193dcbf9ee2158..f8379bdf954bdde122e68585b973f4957ef15739 100644 --- a/tests/script/unique/account/account_len.sim +++ b/tests/script/unique/account/account_len.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/unique/account/authority.sim b/tests/script/unique/account/authority.sim index f88f254a79c83a95ff0e89a3e36eddb2856ad3a2..8f2408de1429a8ea34add79e335f6bf7f42ca2b0 100644 --- a/tests/script/unique/account/authority.sim +++ b/tests/script/unique/account/authority.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/unique/account/basic.sim b/tests/script/unique/account/basic.sim index 0861c9305b836eb96c62acf11a8cad21ec6212a3..00e706a4482d9fa57ed2f97a9995ce84d3667fa1 100644 --- a/tests/script/unique/account/basic.sim +++ b/tests/script/unique/account/basic.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/paras.sim b/tests/script/unique/account/paras.sim index ae511fe2b065dd23d7b1dc19464a06b3e43e1fe0..102f5b6a381e5100b35a4f0125b1318bcb8b1d76 100644 --- a/tests/script/unique/account/paras.sim +++ b/tests/script/unique/account/paras.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/pass_alter.sim b/tests/script/unique/account/pass_alter.sim index bf939acce37cb2d878f1e31ca01e11429d8d4fcb..8b857b014a292d53536c5acf2a00daa15be11239 100644 --- a/tests/script/unique/account/pass_alter.sim +++ b/tests/script/unique/account/pass_alter.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============= step1 diff --git a/tests/script/unique/account/pass_len.sim b/tests/script/unique/account/pass_len.sim index 1dd0a616f291549cf3e431348fe028114f11836d..f4ceb76f7b8b41873217bd11ae2c3d385386b0e9 100644 --- a/tests/script/unique/account/pass_len.sim +++ b/tests/script/unique/account/pass_len.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim index 7fde3652017f5b17e61a39891b1f55c5940e2061..3b9c20b159a6237f469fc1e48b5b3a3f4ca5f7b8 100644 --- a/tests/script/unique/account/usage.sim +++ b/tests/script/unique/account/usage.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start #system sh/exec.sh -n monitor -s 1 system sh/exec.sh -n monitorInterval -s 1 -sleep 3000 +sleep 2000 sql connect print =============== show accounts diff --git a/tests/script/unique/account/user_create.sim b/tests/script/unique/account/user_create.sim index 029bda22ea5a192634c911bba1df74a266402fea..e54a380f0dbef8107de452354ea01bc58262d548 100644 --- a/tests/script/unique/account/user_create.sim +++ b/tests/script/unique/account/user_create.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 diff --git a/tests/script/unique/account/user_len.sim b/tests/script/unique/account/user_len.sim index 91a5ebfaaba09196ff62196b23e60b59784ee1eb..b8d448f0ffc9e43cbc0f0a5a849bda215e72e790 100644 --- a/tests/script/unique/account/user_len.sim +++ b/tests/script/unique/account/user_len.sim @@ -3,7 +3,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $i = 0 diff --git a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim index 915fef47da90a07ec20cc317580306d078321442..7bf2c2ac42a95d1ab6da3ecf50a287eea2ce46bb 100644 --- a/tests/script/unique/arbitrator/check_cluster_cfg_para.sim +++ b/tests/script/unique/arbitrator/check_cluster_cfg_para.sim @@ -90,7 +90,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2~7 and add into cluster diff --git a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim index 6f0f61d2e2f6dd85523f5e9b888172f475d8cbad..dbd0e2bd87c099dca54aa33d609696cb2dc89381 100644 --- a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim +++ b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim @@ -49,7 +49,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -57,7 +57,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 $totalTableNum = 1 $sleepTimer = 3000 @@ -178,7 +178,7 @@ endi print ============== step7: restart dnode3, waiting sync end system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 $loopCnt = 0 wait_dnode3_ready: diff --git a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim index 80a050f883dd4cb1516bee18791addadb5b4567d..e15edb3f3d444b7407244b8cf3fa781ba36357d6 100644 --- a/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim +++ b/tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim @@ -9,11 +9,11 @@ # expect: in dnode2, the files 1837 and 1839 will be removed sql connect -sleep 3000 +sleep 2000 print ============== step7: restart dnode2, waiting sync end system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 wait_dnode2_ready: sql show dnodes diff --git a/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim b/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim index 97433741f53c3053e25b588d7368377e1b890117..057654abb47be64bcba726df8b689b8db4d97f58 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim @@ -39,7 +39,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table to max tables @@ -49,7 +49,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 16 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim b/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim index a6a9129090da41d83e619d6ca123c2ba75d312fd..c597c576e2994725035f83268c6b1f99392627da 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim @@ -40,7 +40,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table to max tables @@ -50,7 +50,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 16 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim index 7bb37c5ebb65a1620f6dbcec24502e9b7e635844..49e1dba06719a3375b221854ccb3a35d1bd03dfe 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $sleepTimer = 3000 @@ -143,7 +143,7 @@ endi print ============== step5: create the middle table 5 and insert data sql create table tb5 using $stb tags( 5 ) -sleep 3000 +sleep 2000 $tsStart = 1420041620000 $i = 5 @@ -212,7 +212,7 @@ endi print ============== step8: create the first table 0 and insert data sql create table tb0 using $stb tags( 0 ) -sleep 3000 +sleep 2000 $tsStart = 1420041640000 $i = 0 @@ -277,7 +277,7 @@ endi print ============== step11: create the last table 9 and insert data sql create table tb9 using $stb tags( 9 ) -sleep 3000 +sleep 2000 $tsStart = 1420041660000 $i = 0 diff --git a/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim index e11dd69598943410def7d75abb53ce2c03fd63c0..2af7cf56b7b547a707abc18f26a18fe74bd8f95e 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim @@ -49,7 +49,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data @@ -59,7 +59,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $rowNum = 10 $tblNum = 16 diff --git a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim index 318a89f96bd25ae8bc8b4f54aa501282b7f7f2ac..96fde9061a0648d69386cb7f5c23450dea6aad7e 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -53,7 +53,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 5000 +sleep 3000 $sleepTimer = 3000 @@ -225,7 +225,7 @@ if $data00 != $totalRows then endi print ============== step5: stop dnode2, and remove its vnode -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s stop -x SIGINT sleep $sleepTimer diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index 9fc19d588ac0928625bee576097770d9c5d7b164..da76cc467b7813586c98761319a01eb1f2ed6bb6 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $sleepTimer = 3000 @@ -359,13 +359,13 @@ endi print ============== step7: stop dnode3/dnode2, and cluster unable to provide services system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 3000 +sleep 2000 sql select count(*) from $stb -x s71 s71: print ============== step8: restart dnode2, and cluster Still unable to provide services system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql select count(*) from $stb -x s81 s81: diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim index 986df81bf695affa71a4e9be740fc246fcc0ee96..c924fee1ae673f4e5696d361be331527d41ff21d 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim @@ -70,7 +70,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster, then create database replica 2, create table , and insert data @@ -78,7 +78,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 $rowNum = 100 $tblNum = 16 @@ -139,7 +139,7 @@ if $data00 != $totalRows then return -1 endi -#sleep 3000 +#sleep 2000 #sql show dnodes #print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 #print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 @@ -175,7 +175,7 @@ endi sql show dnodes if $rows != 3 then - sleep 3000 + sleep 2000 goto wait_drop endi print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 @@ -218,7 +218,7 @@ system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 16 system sh/exec.sh -n dnode3 -s start sql create dnode $hostname3 -sleep 3000 +sleep 2000 $loopCnt = 0 wait_dnode3_ready: @@ -230,7 +230,7 @@ endi sql show dnodes print rows: $rows if $rows != 4 then - sleep 3000 + sleep 2000 goto wait_dnode3_ready endi print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 @@ -271,7 +271,7 @@ sql drop database $db sleep 1000 system sh/exec.sh -n dnode5 -s start sql create dnode $hostname5 -sleep 3000 +sleep 2000 $loopCnt = 0 wait_dnode5: $loopCnt = $loopCnt + 1 @@ -281,7 +281,7 @@ endi sql show dnodes if $rows != 5 then - sleep 3000 + sleep 2000 goto wait_dnode5 endi print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 @@ -352,7 +352,7 @@ endw print info: select count(*) from $stb sleep 2000 sql reset query cache -sleep 3000 +sleep 2000 sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 387ae4625f392b8f108d7e9c34645905348d5e45..b9008e4c42b4f4a34b99bb48d4e38c780e747bb4 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -321,7 +321,7 @@ step9: endi sql show dnodes -if $rows != 2 then +if $rows != 3 then goto step9 endi diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim index 08d2db207b1a0025689c1b21c5d50b1bdd792a9c..73702835f4f41c74ae4161019808d26fd8a7c7a7 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 10000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim index 438e4af34f4a462ea6e84e075889a4e289d35743..27b308411aea5022641ae920a3a4e8848fb1dc04 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim @@ -50,7 +50,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data @@ -60,7 +60,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $rowNum = 10 $tblNum = 16 @@ -152,7 +152,7 @@ endi print ============== step4: restart dnode4, but there are not dnode4 in cluster system sh/exec.sh -n dnode4 -s start -sleep 3000 +sleep 2000 sql show dnodes if $rows != 3 then return -1 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim index 4f80c2389ee25d06a9852d38af9df3d40d838c90..6d81effab63bb52801ab51e53ee1147326b3e851 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim index 3b208e6f1a401f855c434bd02dcab364a05694bb..d22aca07cbd3d3c6a902076103c9759209c6a966 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim index fb0650b78a6875a812fdbb1cbac4a97e48e88348..884a43bce12d1bc2c137bab17b0e780c521e327c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data to can fall disc @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 4 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim index f50081ea70abfc37223e6b3720915ecd4a5eedb9..3c74de49166067624335937d0e0486924b9fdb4f 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim index df41e4df36d747f0f5e066ab249a5c47fb919a19..d0399222f14cc8b8d7ac12ebca8b91549c34942d 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim index d814398d06a8e2f07ef2b5b640b15e542f46f28d..19b29bf342d7c8d045b16111fdc5d2ef9b2039f1 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index 8f2cd9169b5ba43fb0903fe7ffca5e4984b514a1..8e15c4f527db044d52cd725714f15b98161da2ce 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -50,7 +50,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -60,7 +60,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $sleepTimer = 3000 @@ -170,7 +170,7 @@ s31: print ============== step4: restart dnode2, then create database with replica 2, and create table, insert data system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim b/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim index 7906718b0890d6c69a1fb3e00e25ed00191c5b42..d90853d2e41e8384acad2f4dcd5a95eee8298745 100644 --- a/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim +++ b/tests/script/unique/arbitrator/dn3_mn2_killDnode.sim @@ -39,7 +39,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 3, and create table to max tables @@ -47,10 +47,10 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start #system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 -sleep 3000 +sleep 2000 sql create dnode $hostname3 #sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 4 $sleepTimer = 3000 @@ -92,7 +92,7 @@ endi print ============== step3: stop dnode2 system sh/exec.sh -n dnode2 -s stop -sleep 3000 +sleep 2000 sql show mnodes print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 diff --git a/tests/script/unique/arbitrator/insert_duplicationTs.sim b/tests/script/unique/arbitrator/insert_duplicationTs.sim index d58f903dcf2839876b61f0c949aa198423e664cc..4af47ca336c188c3194b9fc64925073f8fb406c2 100644 --- a/tests/script/unique/arbitrator/insert_duplicationTs.sim +++ b/tests/script/unique/arbitrator/insert_duplicationTs.sim @@ -49,7 +49,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -57,7 +57,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 $totalTableNum = 1 $sleepTimer = 3000 @@ -181,7 +181,7 @@ $totalRows = $totalRows + 2 print ============== step6: restart dnode2, waiting sync end system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 $loopCnt = 0 wait_dnode2_ready: $loopCnt = $loopCnt + 1 @@ -213,7 +213,7 @@ endi sleep $sleepTimer # check using select -sleep 5000 +sleep 3000 sql select count(*) from $stb print data00 $data00 if $data00 != $totalRows then diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim index 9527d230e463619df48b84d8f6b3953d4cb81874..0adb6b475938c1aa37f56c20c5d6327c9f89d574 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim index 1da8d749d54e1cd29b3ab00fa28e8406c9fd4d17..a0877ad89c20697e3b9a46f4512766fbd11439d8 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim index 791ba76a8d9af34b3b3dec3c7ce3e2276ddc0a21..376484a0661d89fcd38f6be790437e10c6ef2761 100644 --- a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim index 0d5abb2b9148d51ac6f8424d4f8300218112563e..9f21193400a8d7833afe8dbc5b4671c2f623778d 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim index 0c59ee8525a8bb9fda50e5512aaaf54aeb3e25e3..cb3bbb3a73f5213c346fc675049827730a6e3a01 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim index ce7151f78e85e763f3c3373ba8fce12642fb2514..8a9995f89162883c6529a3cd8e63bc764884147b 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim index e29f8267c31bd96711e5c5bda09e88ff5e416a56..6eed563bbc7c79ac35a782fbd9f2ccaa79f277d6 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim index 8f8b996c09233e72e53ed2224f03bbd417777c8e..2633d822c91d87ad6e5cc70370ac880fd397b889 100644 --- a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim index f3779cf20a5417f3afcc6bb244ce9487664cd874..3abfc401611044b5cc7a1fb9411e4fd461beca78 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim index f70ef1fc8c3f93cfe87629d8e3be0e58422fff83..f2acb8b90a35186395ab559ee428a182a2cba0d4 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim index 8d1a508ef008c8ba6b9fc79612d6d177f7040a90..9d0e967f4e1575195e126cd975540067038409c5 100644 --- a/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim +++ b/tests/script/unique/arbitrator/replica_changeWithArbitrator.sim @@ -60,7 +60,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: replica is 1, and start 1 dnode, then create tables and insert data system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect $totalTableNum = 12 @@ -105,7 +105,7 @@ endi print ============== step2: add 1 new dnode, expect balanced system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 -sleep 3000 +sleep 2000 # expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2 $cnt = 0 @@ -137,7 +137,7 @@ endi print ============== step3: stop dnode1/dnode2, modify cfg numOfMnodes to 2, and restart dnode1/dnode2 system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop -sleep 3000 +sleep 2000 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 @@ -150,7 +150,7 @@ system sh/cfg.sh -n dnode2 -c role -v 0 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print ============= step4: wait dnode ready @@ -193,9 +193,9 @@ if $data00 != $totalRows then endi print ============== step5: stop dnode1 -sleep 5000 -system sh/exec.sh -n dnode1 -s stop sleep 3000 +system sh/exec.sh -n dnode1 -s stop +sleep 2000 $cnt = 0 wait_dnode2_master: diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim index 12db84cbb313e02579d0e689fac3172c92f7bb6a..a8c0e83cc1e5da61a473fa5b9753e90450e57bec 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim index 06d67a1cc9577598f0f8666a718b61d960d40dba..951d26635b257d1c1d3ab81889b66bb909b8038b 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim index b388bc73a66c3529aadd14cdc5f8497a8cfeac87..e4e7f951881cf87c426c869774b2a3a548b29517 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim index 2e20d7b5d488ffe5ff3ed9752e13ed375b795311..0049dc6fba9bd33cd9c2e4f5b56510769dd0c60e 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 #sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index 69cdb9f9424d5253cffe6af72545e881717e040d..499089960100f0be445552112e2b2000164c1ca0 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim index 491b858500e7ae64c2477b9a6ebf4962a9d788a0..10bd4fc8bd3946871605f66493e15cdde28ec257 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica3_createTable.sim b/tests/script/unique/arbitrator/sync_replica3_createTable.sim index d593577bee4bf2ec6997836905a99c327ba6f443..a0b391dd763f96fa6d340fad213943045b45ed69 100644 --- a/tests/script/unique/arbitrator/sync_replica3_createTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_createTable.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 20 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim index 1ef499534fe272a32556a4155a4e622a77ad1ecf..68c6ecbd6ecf7c06b5ae595c16e5b7d4b89435bd 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 20 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim index 7a5966f60c5753e5029f7d107b2b9221d6001714..83e53eaeeb16e0988b778ef0b2515dc12c397d38 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim index abd2d8a788094ada42daf4b843a42a3245b799f0..7496541b76da3245f034924e95fd468b61135aef 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim @@ -45,7 +45,7 @@ system sh/exec_tarbitrator.sh -s start print ============== step1: start dnode1, only deploy mnode system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data @@ -55,7 +55,7 @@ system sh/exec.sh -n dnode4 -s start sql create dnode $hostname2 sql create dnode $hostname3 sql create dnode $hostname4 -sleep 3000 +sleep 2000 $totalTableNum = 10 $sleepTimer = 3000 diff --git a/tests/script/unique/big/maxvnodes.sim b/tests/script/unique/big/maxvnodes.sim index eb17929ff49848acfe0744b4319aeb404a225adc..10dbc8bbff1a7be19791b75d77c705ffb060e48a 100644 --- a/tests/script/unique/big/maxvnodes.sim +++ b/tests/script/unique/big/maxvnodes.sim @@ -17,7 +17,7 @@ system sh/cfg.sh -n dnode2 -c balanceInterval -v 1 print ========== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db blocks 3 cache 1 sql use db @@ -51,7 +51,7 @@ system sh/exec.sh -n dnode2 -s start $x = 0 show3: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 1000 then return -1 endi diff --git a/tests/script/unique/big/restartSpeed.sim b/tests/script/unique/big/restartSpeed.sim index ea4edefda8f96c94aab12d856914565f8e8115f4..bdc4a8678b2c0e766e644a69cd709ecb2cd85268 100644 --- a/tests/script/unique/big/restartSpeed.sim +++ b/tests/script/unique/big/restartSpeed.sim @@ -14,7 +14,7 @@ system sh/cfg.sh -n dnode2 -c walLevel -v 2 print ========== prepare data system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database db blocks 3 cache 1 sql use db diff --git a/tests/script/unique/big/tcp.sim b/tests/script/unique/big/tcp.sim index 3c5cf92c7f183a84b2f1be0ce389bf86f482a42a..b282e2e2223d89dde7cf6ce364b31537593a6cb4 100644 --- a/tests/script/unique/big/tcp.sim +++ b/tests/script/unique/big/tcp.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 30000 system sh/cfg.sh -n dnode1 -c dDebugFlag -v 131 diff --git a/tests/script/unique/cluster/alter.sim b/tests/script/unique/cluster/alter.sim index 7e56707a693e96bd602c012035bebd3b1a8b7b6a..77e040c6cd9675a6e9c0ee9926f056b008bc6596 100644 --- a/tests/script/unique/cluster/alter.sim +++ b/tests/script/unique/cluster/alter.sim @@ -28,11 +28,11 @@ system sh/cfg.sh -n dnode4 -c balance -v 0 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 print ========== step2 sql create database d1 diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index ffd13445edbf76dbc96b82a14848ef6b788237fa..026678af7c5f7d1dfb8924152e8c26ea6dcdb2ec 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -335,8 +335,8 @@ print dnode5 ==> $dnode5Role print ============================== step6 system sh/exec.sh -n dnode1 -s stop -x SIGINT -print stop dnode1 and sleep 10000 -sleep 5000 +print stop dnode1 and sleep 3000 +sleep 3000 sql drop dnode $hostname1 print drop dnode1 and sleep 9000 diff --git a/tests/script/unique/cluster/cache.sim b/tests/script/unique/cluster/cache.sim index e23b40782870be00eb3f010cbcdba693e18b19da..33aaea425c783c8362b4372efad96d384b6dbc70 100644 --- a/tests/script/unique/cluster/cache.sim +++ b/tests/script/unique/cluster/cache.sim @@ -4,8 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 2 system sh/cfg.sh -n dnode2 -c httpMaxThreads -v 2 system sh/cfg.sh -n dnode1 -c monitor -v 1 @@ -17,7 +17,7 @@ system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 system sh/cfg.sh -n dnode2 -c monitorInterval -v 1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create database testdb @@ -33,7 +33,7 @@ while $x < 30 $x = $x + 1 endw -sleep 3000 +sleep 2000 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 @@ -55,3 +55,11 @@ if $rows < 10 then endi #sql create table sys.st as select avg(taosd), avg(system) from sys.cpu interval(30s) + +sql show log.vgroups +if $data05 != master then + return -1 +endi +if $data15 != master then + return -1 +endi diff --git a/tests/script/unique/cluster/cluster_main.sim b/tests/script/unique/cluster/cluster_main.sim index f0a9b1a2147288fb3f0d2437a507296f9517ce09..d3750be6b4cdefa602dc2987c7cab3541589bd8f 100644 --- a/tests/script/unique/cluster/cluster_main.sim +++ b/tests/script/unique/cluster/cluster_main.sim @@ -48,11 +48,11 @@ print ============== step1: start dnode1/dnode2/dnode3 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 print ============== step2: create db1 with replica 3 $db = db1 @@ -84,7 +84,7 @@ sql select count(tbname) from $stb print select count(tbname) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_complete_create_tables endi @@ -93,12 +93,12 @@ print select count(*) from $stb sql select count(*) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_data endi print wait for a while to let clients start insert data -sleep 5000 +sleep 3000 print ============== step4-1: add dnode4/dnode5 into cluster sql create dnode $hostname4 @@ -116,7 +116,7 @@ print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT sleep 10000 #sql drop dnode $hostname1 -#sleep 5000 +#sleep 3000 #system rm -rf ../../../sim/dnode1/data #sleep 20000 @@ -152,7 +152,7 @@ print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql show mnodes print show mnodes diff --git a/tests/script/unique/cluster/cluster_main0.sim b/tests/script/unique/cluster/cluster_main0.sim index 9f775c0cef183bfe3b8dd231a08f5f829fe2d3ef..48403d011b4b7603e861441728fd030de6e3c7ba 100644 --- a/tests/script/unique/cluster/cluster_main0.sim +++ b/tests/script/unique/cluster/cluster_main0.sim @@ -48,11 +48,11 @@ print ============== step1: start dnode1/dnode2/dnode3 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 print ============== step2: create db1 with replica 3 $db = db1 @@ -85,7 +85,7 @@ sql select count(tbname) from $stb print select count(tbname) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_complete_create_tables endi @@ -94,12 +94,12 @@ print select count(*) from $stb sql select count(*) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_data endi print wait for a while to let clients start insert data -sleep 5000 +sleep 3000 $loop_cnt = 0 loop_cluster_do: @@ -110,14 +110,14 @@ system sh/exec.sh -n dnode5 -s start sql create dnode $hostname4 sql create dnode $hostname5 -sleep 5000 +sleep 3000 print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT sleep 10000 #sql drop dnode $hostname1 -#sleep 5000 +#sleep 3000 #system rm -rf ../../../sim/dnode1/data #sleep 20000 print ============== step6-1: restart dnode1 @@ -139,7 +139,7 @@ print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql show mnodes print show mnodes @@ -235,7 +235,7 @@ endi print ============== step14: stop and drop dnode4/dnode5, then remove data dir of dnode4/dnode5 system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT -sleep 3000 +sleep 2000 sql drop dnode $hostname4 sql drop dnode $hostname5 system rm -rf ../../../sim/dnode4/data diff --git a/tests/script/unique/cluster/cluster_main1.sim b/tests/script/unique/cluster/cluster_main1.sim index 7796f1ea00da9d9915aba6a9cbecee0223f99fd4..a2426dc5744fe116329013e658dee48dc957a5f2 100644 --- a/tests/script/unique/cluster/cluster_main1.sim +++ b/tests/script/unique/cluster/cluster_main1.sim @@ -48,11 +48,11 @@ print ============== step1: start dnode1/dnode2/dnode3 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 print ============== step2: create db1 with replica 3 $replica = 3 @@ -92,17 +92,17 @@ print select count(*) from $stb sql select count(*) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_data endi print wait for a while to let clients start insert data -sleep 5000 +sleep 3000 print ============== step4-1: add dnode4/dnode5 into cluster sql create dnode $hostname4 sql create dnode $hostname5 -sleep 5000 +sleep 3000 $loop_cnt = 0 loop_cluster_do: @@ -116,7 +116,7 @@ print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT sleep 10000 #sql drop dnode $hostname1 -#sleep 5000 +#sleep 3000 #system rm -rf ../../../sim/dnode1/data #sleep 20000 print ============== step6-1: restart dnode1 @@ -138,7 +138,7 @@ print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql show mnodes print show mnodes diff --git a/tests/script/unique/cluster/cluster_main2.sim b/tests/script/unique/cluster/cluster_main2.sim index 48661546815a1e4c60e2b8d983f0a3e20e232529..e050ab3acf54d22becb50f6e91f83fc4eab0485b 100644 --- a/tests/script/unique/cluster/cluster_main2.sim +++ b/tests/script/unique/cluster/cluster_main2.sim @@ -48,11 +48,11 @@ print ============== step1: start dnode1/dnode2/dnode3 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 print ============== step2: create db1 with replica 3 $replica = 3 @@ -87,7 +87,7 @@ sql select count(tbname) from $stb print select count(tbname) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_complete_create_tables endi @@ -96,17 +96,17 @@ print select count(*) from $stb sql select count(*) from $stb print data00 $data00 if $data00 < 1000 then - sleep 3000 + sleep 2000 goto wait_subsim_insert_data endi print wait for a while to let clients start insert data -sleep 5000 +sleep 3000 print ============== step4-1: add dnode4/dnode5 into cluster sql create dnode $hostname4 sql create dnode $hostname5 -sleep 5000 +sleep 3000 $loop_cnt = 0 @@ -120,7 +120,7 @@ print ============== step6: stop dnode1 system sh/exec.sh -n dnode1 -s stop -x SIGINT sleep 10000 #sql drop dnode $hostname1 -#sleep 5000 +#sleep 3000 #system rm -rf ../../../sim/dnode1/data #sleep 20000 print ============== step6-1: restart dnode1 @@ -142,7 +142,7 @@ print $data0_9 $data1_9 $data2_9 $data3_9 $data4_9 print ============== step7: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql show mnodes print show mnodes diff --git a/tests/script/unique/cluster/flowctrl.sim b/tests/script/unique/cluster/flowctrl.sim index 6dc60d9fba0fd0a5f7a8088d8277c2da9e06a259..700fa0a3f195c84ed6e7903fe572c81b20b4fd21 100644 --- a/tests/script/unique/cluster/flowctrl.sim +++ b/tests/script/unique/cluster/flowctrl.sim @@ -79,14 +79,14 @@ while $x < 100 endw print =============== step3 -sleep 3000 +sleep 2000 system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT print =============== step4 -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start @@ -101,7 +101,7 @@ endw print =============== step6 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 3000 +sleep 2000 while $x < 300 $ms = $x . s sql insert into tb values (now + $ms , $x ) diff --git a/tests/script/unique/clusterSimCase/cluster_main.sim b/tests/script/unique/clusterSimCase/cluster_main.sim index 39607c2915da7594f7af3562ae3ccd7680b2b4aa..274ce85974bb2238bae60e6ffa9723760ae1c394 100644 --- a/tests/script/unique/clusterSimCase/cluster_main.sim +++ b/tests/script/unique/clusterSimCase/cluster_main.sim @@ -88,8 +88,8 @@ sql connect print ============== step1: add dnode2/dnode3 into cluster sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 -sleep 3000 +sleep 2000 +sleep 2000 print ============== step3: start back client-01.sim #run_back unique/clusterSimCase/client-01.sim #run_back unique/clusterSimCase/client-02.sim @@ -142,7 +142,7 @@ if $vg2Dnode2Status != master then endi -sleep 3000 +sleep 2000 print ============== step3: restart dnode3 system sh/exec.sh -n dnode3 -s start @@ -174,7 +174,7 @@ if $vg2Dnode2Status != master then goto wait_vgroup_chang_1 endi -sleep 3000 +sleep 2000 print ============== step4: stop dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT @@ -207,7 +207,7 @@ if $vg2Dnode2Status != offline then endi -sleep 3000 +sleep 2000 print ============== step5: restart dnode2 system sh/exec.sh -n dnode2 -s start @@ -239,7 +239,7 @@ if $vg2Dnode3Status != master then goto wait_vgroup_chang_3 endi -sleep 3000 +sleep 2000 print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** **** $loop_cnt = $loop_cnt + 1 if $loop_cnt == 50 then diff --git a/tests/script/unique/db/commit.sim b/tests/script/unique/db/commit.sim index caff3c6a7898090b307d487ccdf2b8ac0dd7a1f3..661dd4505f704fff961739075247d4264d617f72 100644 --- a/tests/script/unique/db/commit.sim +++ b/tests/script/unique/db/commit.sim @@ -16,12 +16,12 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 print ========= start dnode1 as master system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ========= start other dnodes sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 print ======== step1 create db sql create database commitdb replica 1 days 7 keep 30 @@ -48,9 +48,9 @@ endi print ======== step2 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql select * from tb order by ts desc print ===> rows $rows @@ -99,9 +99,9 @@ endi print ======== step5 stop dnode system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 sql select * from tb print ===> rows $rows diff --git a/tests/script/unique/db/delete.sim b/tests/script/unique/db/delete.sim index f84339be79ad08951ff60c0df0d8992027c2ef98..c876f23de35f67b0cd2a6f0e428a825d98de7c27 100644 --- a/tests/script/unique/db/delete.sim +++ b/tests/script/unique/db/delete.sim @@ -23,7 +23,7 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 print ======== step1 sql create database db replica 3 blocks 3 @@ -49,7 +49,7 @@ if $rows != 0 then return -1 endi -sleep 3000 +sleep 2000 sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 diff --git a/tests/script/unique/db/replica_reduce32.sim b/tests/script/unique/db/replica_reduce32.sim index a5eb78dacb6065413e06a50d7a0d9f7e28897d9f..ead265d5d54bd415d12147e5b61b04fcc5d7544f 100644 --- a/tests/script/unique/db/replica_reduce32.sim +++ b/tests/script/unique/db/replica_reduce32.sim @@ -169,7 +169,7 @@ sleep 200 print ========= step4 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql insert into d1.t1 values(now, 3) -x s41 s41: diff --git a/tests/script/unique/dnode/alternativeRole.sim b/tests/script/unique/dnode/alternativeRole.sim index af627490ba8671203d740178ae8b42e22b2c3a74..955b757f06df22c884565d0fac350456e79cb73a 100644 --- a/tests/script/unique/dnode/alternativeRole.sim +++ b/tests/script/unique/dnode/alternativeRole.sim @@ -23,14 +23,14 @@ system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 sql show dnodes print dnode1 $data5_1 diff --git a/tests/script/unique/dnode/balance2.sim b/tests/script/unique/dnode/balance2.sim index 7b07eb3f37334e78d1b2828120472d4ea9ebb5a1..4c67e20c3eb5e65b0bacf0aafc3d9dc1a53d5ac3 100644 --- a/tests/script/unique/dnode/balance2.sim +++ b/tests/script/unique/dnode/balance2.sim @@ -120,7 +120,7 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show3: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi diff --git a/tests/script/unique/dnode/balancex.sim b/tests/script/unique/dnode/balancex.sim index da8197e22df73edfd1d5c6f02d014a45511fbae0..d2c738ee97ff8432999125abfd63d253b0546a91 100644 --- a/tests/script/unique/dnode/balancex.sim +++ b/tests/script/unique/dnode/balancex.sim @@ -23,7 +23,7 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 sql create database d1 sql create table d1.t1 (t timestamp, i int) @@ -102,7 +102,7 @@ system sh/exec.sh -n dnode3 -s start $x = 0 show4: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -145,7 +145,7 @@ if $data2_3 != 3 then endi system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql reset query cache sleep 1000 diff --git a/tests/script/unique/dnode/data1.sim b/tests/script/unique/dnode/data1.sim index 61a991148b21b471aef906223a5f600f7db38f5f..75a484abb60c1e09a7c96f8f03902ea4f0f40aa1 100644 --- a/tests/script/unique/dnode/data1.sim +++ b/tests/script/unique/dnode/data1.sim @@ -23,7 +23,7 @@ system sh/cfg.sh -n dnode4 -c wallevel -v 2 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ========== step2 sql create dnode $hostname2 @@ -36,7 +36,7 @@ system sh/exec.sh -n dnode4 -s start $x = 0 show2: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 10 then return -1 endi @@ -71,7 +71,7 @@ sql insert into d1.t1 values(now+5s, 31) $x = 0 show3: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 10 then return -1 endi diff --git a/tests/script/unique/dnode/datatrans_1node.sim b/tests/script/unique/dnode/datatrans_1node.sim index bc38bfaf2df172dfb752eaf94549995ed513ba74..a156c32672cf2936c0b69fe10242744d2dd235a3 100644 --- a/tests/script/unique/dnode/datatrans_1node.sim +++ b/tests/script/unique/dnode/datatrans_1node.sim @@ -5,7 +5,7 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/cfg.sh -n dnode1 -c wallevel -v 2 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print =============== step1 @@ -40,7 +40,7 @@ system sh/exec.sh -n dnode2 -s start print =============== step 4 -sleep 3000 +sleep 2000 sql connect sql select * from db.m1 diff --git a/tests/script/unique/dnode/datatrans_3node.sim b/tests/script/unique/dnode/datatrans_3node.sim index 7c3708c1110df2714db25a59b6a5bc37b5f3818c..7948eb6d3a4317ab3d829a874ced71d9f290d641 100644 --- a/tests/script/unique/dnode/datatrans_3node.sim +++ b/tests/script/unique/dnode/datatrans_3node.sim @@ -31,7 +31,7 @@ system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 print ============== step1: start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -39,7 +39,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 # create table sql drop database -x step1 @@ -76,7 +76,7 @@ system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start print =============== step 4 -sleep 3000 +sleep 2000 sql connect sql select * from db.m1 diff --git a/tests/script/unique/dnode/datatrans_3node_2.sim b/tests/script/unique/dnode/datatrans_3node_2.sim index 4fb3b4535fb640ac7257af33b651941462a56896..844afc5b02f7de4e41d545ad546b9ec41943b18f 100644 --- a/tests/script/unique/dnode/datatrans_3node_2.sim +++ b/tests/script/unique/dnode/datatrans_3node_2.sim @@ -31,7 +31,7 @@ system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 print ============== step1: start dnode1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data @@ -39,7 +39,7 @@ system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start sql create dnode $hostname2 sql create dnode $hostname3 -sleep 3000 +sleep 2000 # create table sql drop database -x step1 @@ -76,7 +76,7 @@ system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode4 -s start print =============== step 4 -sleep 3000 +sleep 2000 sql connect sql select * from db.m1 diff --git a/tests/script/unique/dnode/monitor.sim b/tests/script/unique/dnode/monitor.sim index 1e5b0f6f56c261437ffc25c8a68139c53d7303df..b9b5e41889675ed199b67efc327477271e2f2959 100644 --- a/tests/script/unique/dnode/monitor.sim +++ b/tests/script/unique/dnode/monitor.sim @@ -23,7 +23,7 @@ system sh/cfg.sh -n dnode2 -c monitor -v 1 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 5000 +sleep 3000 sql show dnodes print dnode1 openVnodes $data3_1 @@ -65,7 +65,7 @@ sql select * from log.dn1 print $rows $rows1 = $rows -sleep 3000 +sleep 2000 sql select * from log.dn1 print $rows $rows2 = $rows @@ -79,7 +79,7 @@ sql select * from log.dn2 print $rows $rows1 = $rows -sleep 3000 +sleep 2000 sql select * from log.dn2 print $rows $rows2 = $rows diff --git a/tests/script/unique/dnode/monitor_bug.sim b/tests/script/unique/dnode/monitor_bug.sim index 3169c7cdba4c9f631a41365e25e6537dd823fc5a..efdf5e94b9648a50860dfa79715f6df8cb4c2b3a 100644 --- a/tests/script/unique/dnode/monitor_bug.sim +++ b/tests/script/unique/dnode/monitor_bug.sim @@ -15,7 +15,7 @@ system sh/cfg.sh -n dnode2 -c monitor -v 0 print ========== step1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 5000 +sleep 3000 sql show dnodes print dnode1 openVnodes $data2_1 @@ -64,7 +64,7 @@ sql select * from log.dn1 print $rows $rows1 = $rows -sleep 3000 +sleep 2000 sql select * from log.dn1 print $rows $rows2 = $rows diff --git a/tests/script/unique/dnode/offline1.sim b/tests/script/unique/dnode/offline1.sim index beebbfda60c03d24a774347c8ecf8c2f9a4c6c9e..9bbd14cf07891583711f8298c09da24d9fae4df4 100644 --- a/tests/script/unique/dnode/offline1.sim +++ b/tests/script/unique/dnode/offline1.sim @@ -25,7 +25,7 @@ system sh/exec.sh -n dnode1 -s start sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql show dnodes print dnode1 $data4_1 diff --git a/tests/script/unique/dnode/offline2.sim b/tests/script/unique/dnode/offline2.sim index e8a5460de47919f42187f26a05169fa99ce47dd0..711488cf3f4dc2aebd1980c6615291b0e88dc825 100644 --- a/tests/script/unique/dnode/offline2.sim +++ b/tests/script/unique/dnode/offline2.sim @@ -32,7 +32,7 @@ system sh/exec.sh -n dnode1 -s start sql connect sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start -sleep 3000 +sleep 2000 sql create database d1 replica 2 sql create table d1.t1(ts timestamp, i int) @@ -60,7 +60,7 @@ print dnode1 $data4_2 if $data4_1 != ready then return -1 endi -if $data4_2 != offline then +if $data4_2 == ready then return -1 endi @@ -84,11 +84,11 @@ system sh/exec.sh -n dnode3 -s start system sh/exec.sh -n dnode2 -s start sql drop dnode $hostname2 -sleep 5000 +sleep 3000 $x = 0 show4: $x = $x + 1 - sleep 5000 + sleep 3000 if $x == 10 then return -1 endi diff --git a/tests/script/unique/dnode/remove2.sim b/tests/script/unique/dnode/remove2.sim index f97e55164b14bc62237513a057cff21c36073179..ff92ff7eb05e87cc88a76af2d40502b36f0d0e4f 100644 --- a/tests/script/unique/dnode/remove2.sim +++ b/tests/script/unique/dnode/remove2.sim @@ -120,7 +120,8 @@ sql show dnodes print dnode1 openVnodes $data2_1 print dnode2 openVnodes $data2_2 print dnode3 openVnodes $data2_3 -if $data2_3 != 0 then +# wait dnode3 create first vgroup in dnode-status msg +if $data2_3 != 1 then goto step4 endi @@ -181,4 +182,4 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/dnode/simple.sim b/tests/script/unique/dnode/simple.sim index 014e2dd04f4e221dd036830eca95c50af03bd77e..38d8c08d75b4201fd4375afe50a83d8757ac0120 100644 --- a/tests/script/unique/dnode/simple.sim +++ b/tests/script/unique/dnode/simple.sim @@ -13,7 +13,7 @@ sql create dnode $hostname2 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname3 system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql create database d1 replica 2 sql create table d1.t1 (t timestamp, i int) @@ -44,7 +44,7 @@ endi print ========== step2 sql create dnode $hostname4 system sh/exec.sh -n dnode4 -s start -sleep 3000 +sleep 2000 sql show dnodes print dnode1 openVnodes $data2_1 diff --git a/tests/script/unique/http/admin.sim b/tests/script/unique/http/admin.sim index 8833397487aaca05342b7c41db6c979b6084ca97..1d67a7f86b2aef94a90b74d3d0e6a07dc0cf3ef2 100644 --- a/tests/script/unique/http/admin.sim +++ b/tests/script/unique/http/admin.sim @@ -8,7 +8,7 @@ system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 print ============================ dnode1 start @@ -69,17 +69,17 @@ endi system_content curl -H 'Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.' -d 'show databases' 127.0.0.1:7111/admin/login/root/1 print 7-> $system_content -if $system_content != @{"status":"error","code":4386,"desc":"invalid type of Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi system_content curl -H 'Authorization: Taosd eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJ3d3cudGFvc2RhdGEuY29tIiwicGFzcyI6InRhb3NkYXRhIiwic3ViIjoicm9vdCJ9.xPv3b5odlR7YF8G_QWASjIRbMtA5v4ItToJ35fFgi' 127.0.0.1:7111/admin/login/root/1 print 8-> $system_content -if $system_content != @{"status":"error","code":4389,"desc":"invalid taosd Authorization"}@ then +if $system_content != @{"status":"error","code":4387,"desc":"invalid format of Authorization"}@ then return -1 endi -sleep 3000 +sleep 2000 system_content curl 127.0.0.1:7111/admin/login/root/taosdata print 9 -----> $system_content diff --git a/tests/script/unique/http/opentsdb.sim b/tests/script/unique/http/opentsdb.sim index 526981716537ed8966e23fa3bc98662de0e06854..7d1e6b03d4547a6b0b2a6a7857000a8a6518a002 100644 --- a/tests/script/unique/http/opentsdb.sim +++ b/tests/script/unique/http/opentsdb.sim @@ -5,7 +5,7 @@ system sh/cfg.sh -n dnode1 -c http -v 1 system sh/cfg.sh -n dnode1 -c wallevel -v 0 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect print ============================ dnode1 start @@ -152,7 +152,7 @@ if $system_content != @{"status":"error","code":4515,"desc":"tag value is null"} return -1 endi -sleep 3000 +sleep 2000 print =============== step2 - insert single data system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 1346846400000,"value": 18,"tags": {"host": "web01","group1": "1","dc": "lga"}}]' 127.0.0.1:7111/opentsdb/db/put @@ -169,7 +169,7 @@ endi system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_web01' 127.0.0.1:7111/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",18.000000000]],"rows":1}@ then return -1 endi @@ -186,7 +186,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_cpu_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",18.000000000],["2012-09-05 20:00:05.000",18.000000000]],"rows":2}@ then return -1 endi @@ -194,7 +194,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[3]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[3]],"rows":1}@ then return -1 endi @@ -211,7 +211,7 @@ system_content curl -u root:taosdata -d 'select * from db.sys_mem_d_bbb_lga_1_w print $system_content -if $system_content != @{"status":"succ","head":["ts","value"],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then +if $system_content != @{"status":"succ","head":["ts","value"],"column_meta":[["ts",9,8],["value",7,8]],"data":[["2012-09-05 20:00:00.000",8.000000000],["2012-09-05 20:00:05.000",9.000000000]],"rows":2}@ then return -1 endi @@ -219,7 +219,7 @@ system_content curl -u root:taosdata -d 'select count(*) from db.sys_mem_d_bbb' print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[2]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[2]],"rows":1}@ then return -1 endi @@ -233,7 +233,7 @@ system_content curl -u root:taosdata -d '[{"metric": "sys_cpu","timestamp": 134 system_content curl -u root:taosdata -d 'select count(*) from db.sys_cpu_d_bbb' 127.0.0.1:7111/rest/sql/ print $system_content -if $system_content != @{"status":"succ","head":["count(*)"],"data":[[7]],"rows":1}@ then +if $system_content != @{"status":"succ","head":["count(*)"],"column_meta":[["count(*)",5,8]],"data":[[7]],"rows":1}@ then return -1 endi @@ -244,4 +244,4 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/unique/import/replica3.sim b/tests/script/unique/import/replica3.sim index 714141c412793f6608b11beb0a455ee3d37a3310..5da9e141a5867dd2e3d61114379ede589c0fa744 100644 --- a/tests/script/unique/import/replica3.sim +++ b/tests/script/unique/import/replica3.sim @@ -241,9 +241,9 @@ endi print ================= step13 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print ================= step14 #1520000000000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index 6c9e92502c254c9e1072ed1236822831ce7809cd..8f837b7e477ab801b296b32ddcf9a5c683c351f0 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -159,7 +159,7 @@ if $loopCnt == 20 then endi sql show dnodes -x wait_dnode1_ready -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode1_ready endi @@ -232,7 +232,7 @@ if $loopCnt == 10 then endi sql show dnodes -if $rows != 2 then +if $rows != 3 then sleep 2000 goto wait_dnode2_offline endi diff --git a/tests/script/unique/mnode/mgmt20.sim b/tests/script/unique/mnode/mgmt20.sim index ee9c2b914f237f619f342122a3b1fd2bccc5cf67..8945cffab226ab5dc379057d55e562f5c3ed9cfa 100644 --- a/tests/script/unique/mnode/mgmt20.sim +++ b/tests/script/unique/mnode/mgmt20.sim @@ -68,7 +68,7 @@ if $data2_2 != slave then goto show4 endi -sleep 3000 +sleep 2000 sql select * from log.dn1 $d1_second = $rows sql select * from log.dn2 diff --git a/tests/script/unique/mnode/mgmt22.sim b/tests/script/unique/mnode/mgmt22.sim index 1dc419b17dbd537e09833780c0104e304f8b71fe..399805312ba905d55bceffe011cfe074c831684e 100644 --- a/tests/script/unique/mnode/mgmt22.sim +++ b/tests/script/unique/mnode/mgmt22.sim @@ -46,7 +46,7 @@ print should not drop master print ============== step4 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 sql_error show mnodes print error of no master diff --git a/tests/script/unique/mnode/mgmt26.sim b/tests/script/unique/mnode/mgmt26.sim index 9b534cca06c89d3fa6f05ab82a88fb80514acac2..2816845052e835cf11e0ec7d4ddc71cbdee0ada1 100644 --- a/tests/script/unique/mnode/mgmt26.sim +++ b/tests/script/unique/mnode/mgmt26.sim @@ -90,7 +90,7 @@ print ============== step5 system sh/exec.sh -n dnode2 -s stop system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start sql create dnode $hostname2 sleep 6000 diff --git a/tests/script/unique/mnode/mgmt30.sim b/tests/script/unique/mnode/mgmt30.sim index a9488799334ef81c82e231bc7782993b7a33a4fa..d0858c0d6cdffa1cb1cd7f2ba570ae0521f412d5 100644 --- a/tests/script/unique/mnode/mgmt30.sim +++ b/tests/script/unique/mnode/mgmt30.sim @@ -32,7 +32,7 @@ endi print ============== step2 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 sql create dnode $hostname2 sql create dnode $hostname3 diff --git a/tests/script/unique/mnode/mgmtr2.sim b/tests/script/unique/mnode/mgmtr2.sim index 0c9f961d253e1ca57477f5612ed5f1f28c7b9e26..5afb41905846bff3ce9894e928245a7d34078354 100644 --- a/tests/script/unique/mnode/mgmtr2.sim +++ b/tests/script/unique/mnode/mgmtr2.sim @@ -9,7 +9,7 @@ system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2 print ============== step1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql show mnodes diff --git a/tests/script/unique/stable/dnode2.sim b/tests/script/unique/stable/dnode2.sim index 5c227f8cece29d31232ba1e030dbd30b3698096d..3ca8c4ee20bb74890da8f42c73521186306e3097 100644 --- a/tests/script/unique/stable/dnode2.sim +++ b/tests/script/unique/stable/dnode2.sim @@ -1,8 +1,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/stable/dnode2_stop.sim b/tests/script/unique/stable/dnode2_stop.sim index 19c6de33b3882633acce74ca6c8f9bdbccbf7b1c..c10f04338d894106a6a543a872af69da9e49bae5 100644 --- a/tests/script/unique/stable/dnode2_stop.sim +++ b/tests/script/unique/stable/dnode2_stop.sim @@ -72,7 +72,7 @@ endi sleep 100 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 print =============== step2 sql select count(*) from $mt -x step2 @@ -84,7 +84,7 @@ sql select count(tbcol) from $mt -x step21 step21: system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print =============== step3 sql select count(tbcol) as c from $mt where ts <= 1519833840000 diff --git a/tests/script/unique/stable/dnode3.sim b/tests/script/unique/stable/dnode3.sim index 436ae73595bef47631113498f472d43e173aa7a9..d0708c81542f13634466053d7159c119687dfa04 100644 --- a/tests/script/unique/stable/dnode3.sim +++ b/tests/script/unique/stable/dnode3.sim @@ -2,9 +2,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 diff --git a/tests/script/unique/stream/metrics_balance.sim b/tests/script/unique/stream/metrics_balance.sim index 3cb0d73ab862178e5d648ddd1342e0b930e1f6a0..ff48c2236709635c8d1a790104b0185144a96866 100644 --- a/tests/script/unique/stream/metrics_balance.sim +++ b/tests/script/unique/stream/metrics_balance.sim @@ -8,8 +8,8 @@ system sh/cfg.sh -n dnode1 -c statusInterval -v 1 system sh/cfg.sh -n dnode2 -c statusInterval -v 1 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 @@ -175,7 +175,7 @@ print rows0=>$r0 rows3=>$r3 rows6=>$r6 rows9=>$r9 $x = 0 show1: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -200,7 +200,7 @@ sleep 8000 $x = 0 show2: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi diff --git a/tests/script/unique/stream/metrics_replica1_dnode2.sim b/tests/script/unique/stream/metrics_replica1_dnode2.sim index bbc4d5174cbade0959f5586e2ed15b81bebf4f5b..20c37cefc39f8fa6393d49934adb046f409fca25 100644 --- a/tests/script/unique/stream/metrics_replica1_dnode2.sim +++ b/tests/script/unique/stream/metrics_replica1_dnode2.sim @@ -4,8 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/stream/metrics_replica2_dnode2.sim b/tests/script/unique/stream/metrics_replica2_dnode2.sim index e9944daf3726be0433c4d757c8f6674869baa672..aa8c1871017982cecc695abc8f64d732a8a7fc4e 100644 --- a/tests/script/unique/stream/metrics_replica2_dnode2.sim +++ b/tests/script/unique/stream/metrics_replica2_dnode2.sim @@ -4,8 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sql connect diff --git a/tests/script/unique/stream/metrics_replica2_dnode2_vnoden.sim b/tests/script/unique/stream/metrics_replica2_dnode2_vnoden.sim index f60355cd6ac2037e97ab1fb4bf0ac04895db2955..be2fcefe66ed6ca2e24a44cd22fa072201137b89 100644 --- a/tests/script/unique/stream/metrics_replica2_dnode2_vnoden.sim +++ b/tests/script/unique/stream/metrics_replica2_dnode2_vnoden.sim @@ -4,8 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/stream/metrics_replica2_dnode3.sim b/tests/script/unique/stream/metrics_replica2_dnode3.sim index 981f5e9b70aae8796a29bbc40b05b764b87fa2eb..f7b17610c380d9f90a2cefd4af86ea766facdffa 100644 --- a/tests/script/unique/stream/metrics_replica2_dnode3.sim +++ b/tests/script/unique/stream/metrics_replica2_dnode3.sim @@ -6,9 +6,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 diff --git a/tests/script/unique/stream/metrics_replica3_dnode4.sim b/tests/script/unique/stream/metrics_replica3_dnode4.sim index 902e9db16b6421c209734716d1c97dac00a17c54..402712800313ff5b96f970d12ffe007f77bc26f7 100644 --- a/tests/script/unique/stream/metrics_replica3_dnode4.sim +++ b/tests/script/unique/stream/metrics_replica3_dnode4.sim @@ -8,10 +8,10 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 diff --git a/tests/script/unique/stream/metrics_vnode_stop.sim b/tests/script/unique/stream/metrics_vnode_stop.sim index 1f9cab48e5f738dabe1e07fa5b1017eb23282ac0..cd84cb3cdf5f8096f4986a222cc371db3900f765 100644 --- a/tests/script/unique/stream/metrics_vnode_stop.sim +++ b/tests/script/unique/stream/metrics_vnode_stop.sim @@ -4,8 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/exec.sh -n dnode1 -s start @@ -99,8 +99,8 @@ system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/exec.sh -n dnode2 -s start sleep 2000 @@ -112,11 +112,11 @@ connectTbase2: return -1 endi sql connect -x connectTbase2 -sleep 3000 +sleep 2000 sql create dnode $hostname1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 print ======================== dnode start $dbPrefix = db diff --git a/tests/script/unique/stream/table_balance.sim b/tests/script/unique/stream/table_balance.sim index facb7df45900eff83c18e2a5ee8344d4d426cf9b..45e054e2efdfbd7f3d01e3a860c5ac227f3327fc 100644 --- a/tests/script/unique/stream/table_balance.sim +++ b/tests/script/unique/stream/table_balance.sim @@ -6,8 +6,8 @@ system sh/cfg.sh -n dnode1 -c statusInterval -v 1 system sh/cfg.sh -n dnode2 -c statusInterval -v 1 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 @@ -112,7 +112,7 @@ print rows1=>$r1 rows5=>$r5 rows8=>$r8 $x = 0 show1: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -137,7 +137,7 @@ sleep 8000 $x = 0 show2: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi diff --git a/tests/script/unique/stream/table_move.sim b/tests/script/unique/stream/table_move.sim index fe19e6f402b49ee74f5a4440fb6012925a85161e..964a0c025363fd650e8051312a812fffbddaea7d 100644 --- a/tests/script/unique/stream/table_move.sim +++ b/tests/script/unique/stream/table_move.sim @@ -63,7 +63,7 @@ print ========= start dnode1 system sh/exec.sh -n dnode1 -s start sql connect -sleep 3000 +sleep 2000 $i = 0 $db = $dbPrefix . $i @@ -170,7 +170,7 @@ sleep 8000 $x = 0 show2: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi @@ -199,7 +199,7 @@ sleep 9000 $x = 0 show6: $x = $x + 1 - sleep 3000 + sleep 2000 if $x == 20 then return -1 endi diff --git a/tests/script/unique/stream/table_replica1_dnode2.sim b/tests/script/unique/stream/table_replica1_dnode2.sim index aaab2990b4a80cfe626930cce4c6afc1161c450d..ccc6026e9c92975ccdd4fd12366a11f50a818d3f 100644 --- a/tests/script/unique/stream/table_replica1_dnode2.sim +++ b/tests/script/unique/stream/table_replica1_dnode2.sim @@ -2,8 +2,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/stream/table_replica2_dnode2.sim b/tests/script/unique/stream/table_replica2_dnode2.sim index da24b5ab4e825d2795a7ec1f3e2ccbee563e81ed..947fa0d2f9093c802a9c99c74edddeffca102d38 100644 --- a/tests/script/unique/stream/table_replica2_dnode2.sim +++ b/tests/script/unique/stream/table_replica2_dnode2.sim @@ -4,8 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start sql connect diff --git a/tests/script/unique/stream/table_replica2_dnode2_vnoden.sim b/tests/script/unique/stream/table_replica2_dnode2_vnoden.sim index 0717e6f965fbc3d70caf09102b0fb6df15e76a5e..75300362393eaa543740307d4d11f9a4eabbbc50 100644 --- a/tests/script/unique/stream/table_replica2_dnode2_vnoden.sim +++ b/tests/script/unique/stream/table_replica2_dnode2_vnoden.sim @@ -4,8 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/stream/table_replica2_dnode3.sim b/tests/script/unique/stream/table_replica2_dnode3.sim index 10d9feec53e15b92e01479227d10016c22ed8bb7..49eb3563b3964f05f31d72a8fd1ff12f2b5b3a03 100644 --- a/tests/script/unique/stream/table_replica2_dnode3.sim +++ b/tests/script/unique/stream/table_replica2_dnode3.sim @@ -6,9 +6,9 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 diff --git a/tests/script/unique/stream/table_replica3_dnode4.sim b/tests/script/unique/stream/table_replica3_dnode4.sim index 3b9552084bbc968f6a6076129668208641b0608a..2cc443c72fc656b87ca8c1d330381ed5078cd755 100644 --- a/tests/script/unique/stream/table_replica3_dnode4.sim +++ b/tests/script/unique/stream/table_replica3_dnode4.sim @@ -8,10 +8,10 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 diff --git a/tests/script/unique/stream/table_vnode_stop.sim b/tests/script/unique/stream/table_vnode_stop.sim index 16350949b12e53dd452e1fba46524352a1953408..625de32a8d7a1e5336dd10f313565bdbc0daf0fc 100644 --- a/tests/script/unique/stream/table_vnode_stop.sim +++ b/tests/script/unique/stream/table_vnode_stop.sim @@ -4,8 +4,8 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 system sh/exec.sh -n dnode1 -s start @@ -100,8 +100,8 @@ system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 sleep 2000 system sh/exec.sh -n dnode2 -s start @@ -113,11 +113,11 @@ connectTbase2: return -1 endi sql connect -x connectTbase2 -sleep 3000 +sleep 2000 sql create dnode $hostname1 system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 print ======================== dnode start $dbPrefix = db diff --git a/tests/script/unique/vnode/backup/replica4.sim b/tests/script/unique/vnode/backup/replica4.sim index bccc17e682258b29d08d493a8c457682287ccfa1..c0ff267c734b58e1f198932c86e78c817625981e 100644 --- a/tests/script/unique/vnode/backup/replica4.sim +++ b/tests/script/unique/vnode/backup/replica4.sim @@ -10,10 +10,10 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start $x = 0 diff --git a/tests/script/unique/vnode/backup/replica5.sim b/tests/script/unique/vnode/backup/replica5.sim index d29d11fdafdcf9b368e3f94350fd959168764445..1223cf6b585f7affeb40e2f625a3f6cbd79dae80 100644 --- a/tests/script/unique/vnode/backup/replica5.sim +++ b/tests/script/unique/vnode/backup/replica5.sim @@ -12,11 +12,11 @@ system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode5 -i 5 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 -system sh/cfg.sh -n dnode2 -c walLevel -v 0 -system sh/cfg.sh -n dnode3 -c walLevel -v 0 -system sh/cfg.sh -n dnode4 -c walLevel -v 0 -system sh/cfg.sh -n dnode5 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 +system sh/cfg.sh -n dnode5 -c walLevel -v 1 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/unique/vnode/many.sim b/tests/script/unique/vnode/many.sim index 869c7f82953e99ef9e14a193f3894c4b53466b9e..a9298b1cf275c24ab6ebe7fea9387a51d6d044ba 100644 --- a/tests/script/unique/vnode/many.sim +++ b/tests/script/unique/vnode/many.sim @@ -80,7 +80,7 @@ $lastRows4 = $rows print ======== step2 run_back unique/vnode/back_insert_many.sim -sleep 5000 +sleep 3000 print ======== step3 @@ -89,15 +89,15 @@ loop: print ======== step4 system sh/exec.sh -n dnode3 -s stop -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 print ======== step5 system sh/exec.sh -n dnode2 -s stop -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print ======== step6 sql select count(*) from db1.tb1 diff --git a/tests/script/unique/vnode/replica2_a_large.sim b/tests/script/unique/vnode/replica2_a_large.sim index 801570dd9c18daa5611d621660e88905fa73c2b3..6f318038454a171445600e37d17852b2c2de835e 100644 --- a/tests/script/unique/vnode/replica2_a_large.sim +++ b/tests/script/unique/vnode/replica2_a_large.sim @@ -29,14 +29,14 @@ print ============== step0: start tarbitrator system sh/exec_tarbitrator.sh -s start system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start -t system sh/exec.sh -n dnode3 -s start -t -sleep 3000 +sleep 2000 print ========= step1 sql create database db replica 2 @@ -56,7 +56,7 @@ $lastRows = $rows print ======== step2 #run_back unique/vnode/back_insert_many.sim run_back unique/vnode/back_insert.sim -sleep 3000 +sleep 2000 print ======== step3 diff --git a/tests/script/unique/vnode/replica2_basic2.sim b/tests/script/unique/vnode/replica2_basic2.sim index 6976353f3e292b3752b1df9f7ac1c398087a644a..c081f878dda513a3b11c9546db4dad31c3d2c5ff 100644 --- a/tests/script/unique/vnode/replica2_basic2.sim +++ b/tests/script/unique/vnode/replica2_basic2.sim @@ -22,13 +22,13 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 print ========= start dnodes system sh/exec.sh -n dnode1 -s start -sleep 3000 +sleep 2000 sql connect sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 sql reset query cache @@ -126,7 +126,7 @@ endi print ========= step3 system sh/exec.sh -n dnode2 -s stop -x SIGINT -sleep 5000 +sleep 3000 #sql insert into d1.t1 values(now, 3) #sql insert into d2.t2 values(now, 3) @@ -155,9 +155,9 @@ sleep 5000 print ========= step4 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode3 -s stop -x SIGINT -sleep 5000 +sleep 3000 #sql insert into d1.t1 values(now, 4) #sql insert into d2.t2 values(now, 4) @@ -186,7 +186,7 @@ sleep 5000 print ========= step5 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 sql insert into d1.t1 values(now, 5) sql insert into d2.t2 values(now, 5) diff --git a/tests/script/unique/vnode/replica2_repeat.sim b/tests/script/unique/vnode/replica2_repeat.sim index 5dbb24437d3dc534018b788c5be1e29bf32cb72d..ac68d591648b2dd66f3fdda8c70b0af40c814459 100644 --- a/tests/script/unique/vnode/replica2_repeat.sim +++ b/tests/script/unique/vnode/replica2_repeat.sim @@ -65,7 +65,7 @@ $lastRows = $rows print ======== step2 run_back unique/vnode/back_insert.sim -sleep 3000 +sleep 2000 print ======== step3 @@ -74,15 +74,15 @@ loop: print ======== step4 system sh/exec.sh -n dnode2 -s stop -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 print ======== step5 system sh/exec.sh -n dnode3 -s stop -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 print ======== step6 sql select count(*) from db.tb diff --git a/tests/script/unique/vnode/replica3_repeat.sim b/tests/script/unique/vnode/replica3_repeat.sim index d866fb05d89b5441d83390a154324fb4290c451d..636bc64f89948a8bd4ec0d3dc9dc2a1b10f50e00 100644 --- a/tests/script/unique/vnode/replica3_repeat.sim +++ b/tests/script/unique/vnode/replica3_repeat.sim @@ -72,32 +72,32 @@ $lastRows = $rows print ======== step2 run_back unique/vnode/back_insert.sim -sleep 3000 +sleep 2000 print ======== step3 system sh/exec.sh -n dnode2 -s stop -sleep 5000 +sleep 3000 $x = 0 loop: print ======== step4 system sh/exec.sh -n dnode2 -s start -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode3 -s stop -sleep 5000 +sleep 3000 print ======== step5 system sh/exec.sh -n dnode3 -s start -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode4 -s stop -sleep 5000 +sleep 3000 print ======== step6 system sh/exec.sh -n dnode4 -s start -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode2 -s stop -sleep 5000 +sleep 3000 print ======== step7 sql select count(*) from db.tb diff --git a/tests/script/unique/vnode/replica3_vgroup.sim b/tests/script/unique/vnode/replica3_vgroup.sim index 11295ba0a42cfea1ee6ff5fd50f8563ee344fc77..de4c48eca5683fc7599d545e2887ae8d2c073b82 100644 --- a/tests/script/unique/vnode/replica3_vgroup.sim +++ b/tests/script/unique/vnode/replica3_vgroup.sim @@ -16,13 +16,13 @@ sql create dnode $hostname2 sql create dnode $hostname3 system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start -sleep 3000 +sleep 2000 $N = 10 $table = table_r3 $db = db1 -sleep 3000 +sleep 2000 print =================== step 1 diff --git a/tests/script/windows/alter/metrics.sim b/tests/script/windows/alter/metrics.sim index 7d5dc779490f7591ff9330c47622e46f98fb1783..7dfda05bd0b0849fa6cbdb27e064113ad2fec595 100644 --- a/tests/script/windows/alter/metrics.sim +++ b/tests/script/windows/alter/metrics.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases @@ -368,9 +368,9 @@ endi print ======== step9 print ======== step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use d2 sql describe tb diff --git a/tests/script/windows/alter/table.sim b/tests/script/windows/alter/table.sim index 03182e613d6cea983529e75512b736acfd702765..52757da20ef226ececec64698cea7846584a73bd 100644 --- a/tests/script/windows/alter/table.sim +++ b/tests/script/windows/alter/table.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases @@ -319,9 +319,9 @@ endi print ======== step9 print ======== step10 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 5000 +sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 5000 +sleep 3000 sql use d1 sql describe tb diff --git a/tests/script/windows/compute/avg.sim b/tests/script/windows/compute/avg.sim index b655abf1638b290bc8b6c5dd4ac8887da57c8545..7445808db5dbd293e781f66ed7010396413880bc 100644 --- a/tests/script/windows/compute/avg.sim +++ b/tests/script/windows/compute/avg.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/compute/bottom.sim b/tests/script/windows/compute/bottom.sim index dc104a8ebd4c92587f26620df0c13438b2c597bf..18f6c0bd09d1e6ee4cba734dab20f2634e587d32 100644 --- a/tests/script/windows/compute/bottom.sim +++ b/tests/script/windows/compute/bottom.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/compute/count.sim b/tests/script/windows/compute/count.sim index 9c9d8821b01c5fa86f8eefe850c0923f2681d12b..227b49a8bd61d74f242f3f505fddd599a2862cdc 100644 --- a/tests/script/windows/compute/count.sim +++ b/tests/script/windows/compute/count.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/compute/diff.sim b/tests/script/windows/compute/diff.sim index 667fcdbcff7a92c130ee5fe2253f25fd636790c3..bdc5a0e3d8a781bdccc500ef667ad6f568ad8e66 100644 --- a/tests/script/windows/compute/diff.sim +++ b/tests/script/windows/compute/diff.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/first.sim b/tests/script/windows/compute/first.sim index d6e1b1deeaccff9fa5199eed86b1d85d7c541c29..a83939c2a629e13eb474b6b936e7e4b409d7329e 100644 --- a/tests/script/windows/compute/first.sim +++ b/tests/script/windows/compute/first.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/interval.sim b/tests/script/windows/compute/interval.sim index 4bf548ccf21ad1cf89d6688d1921b9742becbe1f..feacce16066a4b5a06268033dedefe327a9f6fa8 100644 --- a/tests/script/windows/compute/interval.sim +++ b/tests/script/windows/compute/interval.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/last.sim b/tests/script/windows/compute/last.sim index 63d4d3ecbdd9cd000e7a10a814aa19bb741c6880..379a5ae64a48b467ffdae816066e8bc3c956d50c 100644 --- a/tests/script/windows/compute/last.sim +++ b/tests/script/windows/compute/last.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/leastsquare.sim b/tests/script/windows/compute/leastsquare.sim index 69c8fb377b00ad28013ac81f07818151bf387791..20353a409e166243212c072d5596492e251e984b 100644 --- a/tests/script/windows/compute/leastsquare.sim +++ b/tests/script/windows/compute/leastsquare.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/max.sim b/tests/script/windows/compute/max.sim index e480736550781babf036a3ab4af40f81753dc8b6..641b31fe362712e09081f105bacdb11f41f2bb99 100644 --- a/tests/script/windows/compute/max.sim +++ b/tests/script/windows/compute/max.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/min.sim b/tests/script/windows/compute/min.sim index 1ff637cecd27c979bb7c4a978d806726c8c62ff6..3528f573ce4c65ac03ac0c0d6c5502b33979cb17 100644 --- a/tests/script/windows/compute/min.sim +++ b/tests/script/windows/compute/min.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/percentile.sim b/tests/script/windows/compute/percentile.sim index 5e327055a8bfd563691c04ff5cb9ea7fbd9dddc5..fa6212f013b4ec0aa92c06842263fd32be05e1e9 100644 --- a/tests/script/windows/compute/percentile.sim +++ b/tests/script/windows/compute/percentile.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/stddev.sim b/tests/script/windows/compute/stddev.sim index 2aa481248ae511ae0aaa3e2621c05eb9d4ea71e2..eea6c8aa0529d0f8dba9907de76017751fbf3d6f 100644 --- a/tests/script/windows/compute/stddev.sim +++ b/tests/script/windows/compute/stddev.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/sum.sim b/tests/script/windows/compute/sum.sim index 30e98a5b2596306865d774267936e4380a16d2d1..a429ce99e0b2faea0d594cc97c7f232b11c5e781 100644 --- a/tests/script/windows/compute/sum.sim +++ b/tests/script/windows/compute/sum.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/compute/top.sim b/tests/script/windows/compute/top.sim index 9590997ef76b396f37102b46140b8ff58598fb9e..65e448b0fa5fb1dffcaf1e63129099c57a4a9a02 100644 --- a/tests/script/windows/compute/top.sim +++ b/tests/script/windows/compute/top.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/db/basic.sim b/tests/script/windows/db/basic.sim index fffde94d66f1ae30b7632c66ff6444cfe3ba1934..914e456fe1e3a52ca107275b30995f88c37b0415 100644 --- a/tests/script/windows/db/basic.sim +++ b/tests/script/windows/db/basic.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/db/len.sim b/tests/script/windows/db/len.sim index 5afa2496dde23b624b6d54afd18f096c59d47f46..3356165117d3986d9eef4e7dced3bb1b61dd36da 100644 --- a/tests/script/windows/db/len.sim +++ b/tests/script/windows/db/len.sim @@ -1,4 +1,4 @@ -sleep 3000 +sleep 2000 sql connect sql show databases diff --git a/tests/script/windows/field/2.sim b/tests/script/windows/field/2.sim index 8ac6fa1a1b2dd85715b536efdb14dc8b6e6d79a9..e258fbe80efbdde2d1b8f289f5302b74adb8b0f7 100644 --- a/tests/script/windows/field/2.sim +++ b/tests/script/windows/field/2.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/3.sim b/tests/script/windows/field/3.sim index 331e930b3186ad9541912f87e5342046046850cf..e3fe0b0b1162548ccd8ce0d4e5d5606a628abcef 100644 --- a/tests/script/windows/field/3.sim +++ b/tests/script/windows/field/3.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/4.sim b/tests/script/windows/field/4.sim index c6224c46ee2c0064c5fa5ea14a67310fee11d423..aabfe2be2ce76d5fa52fcde85c129b0f4bdf75ac 100644 --- a/tests/script/windows/field/4.sim +++ b/tests/script/windows/field/4.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/5.sim b/tests/script/windows/field/5.sim index d1f40059d027fba7797967c7b65d86457f4fd3b9..874730ff1c9cd341e8c3cc1f9c8d33b228d5b3ad 100644 --- a/tests/script/windows/field/5.sim +++ b/tests/script/windows/field/5.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/6.sim b/tests/script/windows/field/6.sim index 98071f87df11ccd6d5984ce9065485bf2931a6ee..77277df35b2a41bbf7ab759763b9601054b0d5d6 100644 --- a/tests/script/windows/field/6.sim +++ b/tests/script/windows/field/6.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/bigint.sim b/tests/script/windows/field/bigint.sim index bef571f4456dc6a05d6844b28128df4137c39bf4..f912ee968b0955e339340e9ee712b99eb954914a 100644 --- a/tests/script/windows/field/bigint.sim +++ b/tests/script/windows/field/bigint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/binary.sim b/tests/script/windows/field/binary.sim index 72a356e684bbada0ec1b81ca58303a3f885eca4f..aa641878e4c49463aca22ab3f57b43a558e955f7 100644 --- a/tests/script/windows/field/binary.sim +++ b/tests/script/windows/field/binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/bool.sim b/tests/script/windows/field/bool.sim index abc970264dae1346bb2177516b4056c024b69da2..3ef56a1d95082faf54f1e34db4009b8af0e80c6c 100644 --- a/tests/script/windows/field/bool.sim +++ b/tests/script/windows/field/bool.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/double.sim b/tests/script/windows/field/double.sim index e805e0373b83ab440a846139681d040a6c4b7ad1..ef404d28dc306edaf2fe6be00eb83dfb484c23a3 100644 --- a/tests/script/windows/field/double.sim +++ b/tests/script/windows/field/double.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/float.sim b/tests/script/windows/field/float.sim index 4178ab4e1e260ba9251466c53c0cd4da10ee0dc3..8435f31c1fbc3353f87f8abc620184919b32ccbc 100644 --- a/tests/script/windows/field/float.sim +++ b/tests/script/windows/field/float.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/int.sim b/tests/script/windows/field/int.sim index 05dc19094d85110672201436f99f19c19b425db8..781b9394849192725ba5d46e3d669d53838501b7 100644 --- a/tests/script/windows/field/int.sim +++ b/tests/script/windows/field/int.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/single.sim b/tests/script/windows/field/single.sim index 6422b7f6971870d72b6272c77d6109484501eb5e..b6b44020915e57b130c77f2c40e1a7a2e4e1342c 100644 --- a/tests/script/windows/field/single.sim +++ b/tests/script/windows/field/single.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/smallint.sim b/tests/script/windows/field/smallint.sim index 8bf41f45a5e62e584b39daa1ecde34691a922eb2..5f1839226b1d3aabfebfe9b30ad3f8b02d1e21a9 100644 --- a/tests/script/windows/field/smallint.sim +++ b/tests/script/windows/field/smallint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/field/tinyint.sim b/tests/script/windows/field/tinyint.sim index 16c19ba38d58c407fd91ff43654b502fe9bb2d36..c90ff3f932a3eeb52e1ff607ac0bcacf21fecfe4 100644 --- a/tests/script/windows/field/tinyint.sim +++ b/tests/script/windows/field/tinyint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/import/basic.sim b/tests/script/windows/import/basic.sim index 491b4f8b3445e616d75da1ca70058b1fe49089f9..ee19893ae21fd8c97eeb9aa5898ead86b8b8d19f 100644 --- a/tests/script/windows/import/basic.sim +++ b/tests/script/windows/import/basic.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/basic.sim b/tests/script/windows/insert/basic.sim index 54cbd3f4d9a8ee068ff4595f15f0642ca3e92674..a516f80e1013a40a8889ca939ca650360e4fc976 100644 --- a/tests/script/windows/insert/basic.sim +++ b/tests/script/windows/insert/basic.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_block1_file.sim b/tests/script/windows/insert/query_block1_file.sim index 388ed061e5c583ec46113811542b2f87267b860e..62b74ac19c18a624d41941e8c51a3e8fb87ddee0 100644 --- a/tests/script/windows/insert/query_block1_file.sim +++ b/tests/script/windows/insert/query_block1_file.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_block1_memory.sim b/tests/script/windows/insert/query_block1_memory.sim index 9e4fc68d093d07d5c43797efc28265de750c376c..6c3e58d70e992a79bfcd84a670616b9b3441393f 100644 --- a/tests/script/windows/insert/query_block1_memory.sim +++ b/tests/script/windows/insert/query_block1_memory.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_block2_file.sim b/tests/script/windows/insert/query_block2_file.sim index 9fd443447612917fd0ba85a9d8a591a7a384b87c..164c8a1124eb7648c2cb0d39a1dba3839849aab4 100644 --- a/tests/script/windows/insert/query_block2_file.sim +++ b/tests/script/windows/insert/query_block2_file.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_block2_memory.sim b/tests/script/windows/insert/query_block2_memory.sim index ede7f3efc6e807492845786f5979e14a6ba337c6..f3ceb503acfebae398856d1a2d085fd0e3a0ce2d 100644 --- a/tests/script/windows/insert/query_block2_memory.sim +++ b/tests/script/windows/insert/query_block2_memory.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_file_memory.sim b/tests/script/windows/insert/query_file_memory.sim index 083beb4ac5f3983eb90a0d38811398dff510c0a7..d9752c40c9cc7c264a86e245eb3877fc47783b2a 100644 --- a/tests/script/windows/insert/query_file_memory.sim +++ b/tests/script/windows/insert/query_file_memory.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/insert/query_multi_file.sim b/tests/script/windows/insert/query_multi_file.sim index 465970f9426440262f2a694c510ba40d184f7f64..ba617ce63c6886bc978fe732222a54a7a99ad4d5 100644 --- a/tests/script/windows/insert/query_multi_file.sim +++ b/tests/script/windows/insert/query_multi_file.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/binary.sim b/tests/script/windows/table/binary.sim index 64a081c72f5e4e5bd26013df03d7667211b4dce5..5efa0bc6667c4f2418e53f341315bf6529527455 100644 --- a/tests/script/windows/table/binary.sim +++ b/tests/script/windows/table/binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/bool.sim b/tests/script/windows/table/bool.sim index 9486c422215f39fac909af7620452dd3443eef8d..0d185d31e81c34d7289d753cce63ec419612646b 100644 --- a/tests/script/windows/table/bool.sim +++ b/tests/script/windows/table/bool.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/column_name.sim b/tests/script/windows/table/column_name.sim index fffb1334e5e317e6d22155abf4624463bfce9989..6f9f954461c6faaddb01ab8e327c008a86809bbd 100644 --- a/tests/script/windows/table/column_name.sim +++ b/tests/script/windows/table/column_name.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/column_num.sim b/tests/script/windows/table/column_num.sim index d182696ce08176f7d19c74858c069f1cdf73dd4c..395ee02cdea0b90fe8cd248af63cc950c090c283 100644 --- a/tests/script/windows/table/column_num.sim +++ b/tests/script/windows/table/column_num.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/column_value.sim b/tests/script/windows/table/column_value.sim index c59e7af8baae9c0b9b5140a624bd9b4c0c97c385..8db85f16ad876deff5c1a4b445351c83e9c5a3b7 100644 --- a/tests/script/windows/table/column_value.sim +++ b/tests/script/windows/table/column_value.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/db.table.sim b/tests/script/windows/table/db.table.sim index 97a9e6fbe94db8fec01db3db5e0a0c0335788842..0e207c998276fe4e33d0f875155d971506aadb70 100644 --- a/tests/script/windows/table/db.table.sim +++ b/tests/script/windows/table/db.table.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/double.sim b/tests/script/windows/table/double.sim index 93bf3bb1490d21d18c56bea0fc2e2a6535d59b43..b88ac4a199c9394422c7427eda6d824ec524e667 100644 --- a/tests/script/windows/table/double.sim +++ b/tests/script/windows/table/double.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/float.sim b/tests/script/windows/table/float.sim index 684f78a386f886b3f2c6f1c2e8510ebe2a1a043f..741525830d9a5ea38d82420e9affcb8015dab1db 100644 --- a/tests/script/windows/table/float.sim +++ b/tests/script/windows/table/float.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/table/table.sim b/tests/script/windows/table/table.sim index 985620152afae21b4c7c239bb80196bbe8facca2..13d157627720e9f5c8a355255782b10ea43b4a58 100644 --- a/tests/script/windows/table/table.sim +++ b/tests/script/windows/table/table.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ============================ dnode1 start sql show databases diff --git a/tests/script/windows/table/table_len.sim b/tests/script/windows/table/table_len.sim index 367f1c6895c80fb019a68bc36f0921becf1f30c7..72ed549466a61e984a0d9f287639a590c7d07f79 100644 --- a/tests/script/windows/table/table_len.sim +++ b/tests/script/windows/table/table_len.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/tag/3.sim b/tests/script/windows/tag/3.sim index 63a8766727fbdc14ce4504949a48bad7be6b5f64..5479be158ba4ab0be883d1f0eb26d2eb09aaafa1 100644 --- a/tests/script/windows/tag/3.sim +++ b/tests/script/windows/tag/3.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/4.sim b/tests/script/windows/tag/4.sim index 7e9af7ece7ebc8f3548dcacd3c26b094a9990160..17552010b0382db3e9cc7e0bafdecdefff03dd96 100644 --- a/tests/script/windows/tag/4.sim +++ b/tests/script/windows/tag/4.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/5.sim b/tests/script/windows/tag/5.sim index 5dc128a0e0eb68eb2a4c227ade718708224c5f8c..f06d78e7b5b4082699349e12ee99c96e110fa91b 100644 --- a/tests/script/windows/tag/5.sim +++ b/tests/script/windows/tag/5.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/6.sim b/tests/script/windows/tag/6.sim index 12e9c597f0deb9ec00ef70ae9cccb761f79cfbc6..64cb9df6f030d4c0314d533fb6239daf7ba3586e 100644 --- a/tests/script/windows/tag/6.sim +++ b/tests/script/windows/tag/6.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/add.sim b/tests/script/windows/tag/add.sim index 0a1416b68c34245a1ef6553a7d19b7601b34de79..02d027ccf4e43c472a1d8b60daa99d254b2df35c 100644 --- a/tests/script/windows/tag/add.sim +++ b/tests/script/windows/tag/add.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/bigint.sim b/tests/script/windows/tag/bigint.sim index d988ad1fdc6dca3616785a7646efc6fbf9f0d466..ebb67d452c82586d31f673d7c9c5c78249285795 100644 --- a/tests/script/windows/tag/bigint.sim +++ b/tests/script/windows/tag/bigint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/binary.sim b/tests/script/windows/tag/binary.sim index 9dc18cfa94bba50e748edc48beabc40f9445243f..c59039b6a6d8ba5d05a653c1f5d40de170e89f15 100644 --- a/tests/script/windows/tag/binary.sim +++ b/tests/script/windows/tag/binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/binary_binary.sim b/tests/script/windows/tag/binary_binary.sim index ba688aa80e234e4ffcf117154d0ff45827ddb02f..361a6edb8b3f6d9440a1796bed5ca7c431bcf115 100644 --- a/tests/script/windows/tag/binary_binary.sim +++ b/tests/script/windows/tag/binary_binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/bool.sim b/tests/script/windows/tag/bool.sim index a7e5d909c598bc748bd431ba2a1c5d0e4db15ec6..adf12338d089c41fea9e23b65b8dee03f8eb5b46 100644 --- a/tests/script/windows/tag/bool.sim +++ b/tests/script/windows/tag/bool.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/bool_binary.sim b/tests/script/windows/tag/bool_binary.sim index 639f6c5f2f6f80ffe9c9237c0352c4951a1155a3..064677ee407ee9851e45ea3c743b5d95c66e8545 100644 --- a/tests/script/windows/tag/bool_binary.sim +++ b/tests/script/windows/tag/bool_binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/bool_int.sim b/tests/script/windows/tag/bool_int.sim index 900cc9e8a181584cbabb5cee2cd7e048449f7f4f..ef5cd27553161345e95d1ca1b0f414ed16394cb1 100644 --- a/tests/script/windows/tag/bool_int.sim +++ b/tests/script/windows/tag/bool_int.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/change.sim b/tests/script/windows/tag/change.sim index 75a976bbb1553960bfd77f6470e8cddabf9db110..4126ea1181e8b554cac2833eb66c27f3d4290071 100644 --- a/tests/script/windows/tag/change.sim +++ b/tests/script/windows/tag/change.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases @@ -175,7 +175,7 @@ sql alter table $mt change tag tgcol3 tgcol9 sql alter table $mt change tag tgcol5 tgcol10 sql alter table $mt change tag tgcol6 tgcol11 -sleep 5000 +sleep 3000 sql reset query cache print =============== step2 diff --git a/tests/script/windows/tag/column.sim b/tests/script/windows/tag/column.sim index 9f5bfce07e724d6aaec5268f6d85a67805f92209..40159bcae347e1292e2c327110bc4db6b7176bbb 100644 --- a/tests/script/windows/tag/column.sim +++ b/tests/script/windows/tag/column.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/create.sim b/tests/script/windows/tag/create.sim index 6a76c93d83db0afa3e9e9b2de00d614fad66d9e6..62dc8a7a21c442b0a38fd067da0d55ad5c8ed8ef 100644 --- a/tests/script/windows/tag/create.sim +++ b/tests/script/windows/tag/create.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/delete.sim b/tests/script/windows/tag/delete.sim index 9e8ea9aba03c99d116d0923bd27006c34c0253a2..2b503fdf47ac2eaa5ea18c8c6af1bbbe05e9c44c 100644 --- a/tests/script/windows/tag/delete.sim +++ b/tests/script/windows/tag/delete.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases @@ -409,7 +409,7 @@ sql alter table $mt drop tag tgcol3 sql alter table $mt drop tag tgcol4 sql alter table $mt drop tag tgcol6 -sleep 5000 +sleep 3000 print =============== step2 $i = 2 diff --git a/tests/script/windows/tag/double.sim b/tests/script/windows/tag/double.sim index 5445b1dbea083ea6664350c66b6342e606213678..4381aa20f939d79ac81804a5be54952db7a27ba4 100644 --- a/tests/script/windows/tag/double.sim +++ b/tests/script/windows/tag/double.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/filter.sim b/tests/script/windows/tag/filter.sim index f704f32cd2b5ead591ef76ed2237099273ba95a4..802e9a312f8cccbc34bb537dbba5ec3ab13b11d5 100644 --- a/tests/script/windows/tag/filter.sim +++ b/tests/script/windows/tag/filter.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/float.sim b/tests/script/windows/tag/float.sim index 64424c1e209bea2fc3a8c3aaa20dec4e03539587..8df44c24a56d8c013af7c63aa9d45b720069cbaa 100644 --- a/tests/script/windows/tag/float.sim +++ b/tests/script/windows/tag/float.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/int.sim b/tests/script/windows/tag/int.sim index 7d7b5271d194df316b1cc1ac6cd4db9e45bbe2ff..dbff8c15b6aad716a1c90ad5c74269f955c7307d 100644 --- a/tests/script/windows/tag/int.sim +++ b/tests/script/windows/tag/int.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/int_binary.sim b/tests/script/windows/tag/int_binary.sim index 1dd4771605046b8b54d89d54155f41fa9f751b69..94aa9eb7f40441f5cc4e399ac8a4f5f5ce396beb 100644 --- a/tests/script/windows/tag/int_binary.sim +++ b/tests/script/windows/tag/int_binary.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/int_float.sim b/tests/script/windows/tag/int_float.sim index cdb9032d8cc39780f7b71297901914e86bfc1ad2..9789c9ea06640432e8866a787ec0389ed3d239c2 100644 --- a/tests/script/windows/tag/int_float.sim +++ b/tests/script/windows/tag/int_float.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/set.sim b/tests/script/windows/tag/set.sim index 16103c6ce86cf62c8e8a228bebeaf931190435c4..54b87c7d0c1c4eac55a16d42b61979329a31167d 100644 --- a/tests/script/windows/tag/set.sim +++ b/tests/script/windows/tag/set.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/smallint.sim b/tests/script/windows/tag/smallint.sim index dbab4f2d43c86578c7726ca8db23863ec24f335a..bc668b164d53309512d69434e021f5744da2a253 100644 --- a/tests/script/windows/tag/smallint.sim +++ b/tests/script/windows/tag/smallint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/tag/tinyint.sim b/tests/script/windows/tag/tinyint.sim index 7a0237c0d9dd9a02845774ad5102226955c83b38..44fc9ba4dc0d0f4a670c3c03abf36a38f658f878 100644 --- a/tests/script/windows/tag/tinyint.sim +++ b/tests/script/windows/tag/tinyint.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 print ======================== dnode1 start sql show databases diff --git a/tests/script/windows/vector/metrics_field.sim b/tests/script/windows/vector/metrics_field.sim index e7c926e141bc90531adea7dac389a7c9f59d7fdf..dfaa7e1d9978357f5858e529ebac40941dd6bf2c 100644 --- a/tests/script/windows/vector/metrics_field.sim +++ b/tests/script/windows/vector/metrics_field.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/metrics_mix.sim b/tests/script/windows/vector/metrics_mix.sim index 3d94a9638520961119740ba14628dca4143d6040..111fdebb05735057a922ce313d9c595a7d8491ff 100644 --- a/tests/script/windows/vector/metrics_mix.sim +++ b/tests/script/windows/vector/metrics_mix.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/metrics_query.sim b/tests/script/windows/vector/metrics_query.sim index c292c6b306da676dfcc7e94fc8db30b1533848db..45e734f468deb4fb1567260fdde8b072bdb2a3b2 100644 --- a/tests/script/windows/vector/metrics_query.sim +++ b/tests/script/windows/vector/metrics_query.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/metrics_tag.sim b/tests/script/windows/vector/metrics_tag.sim index f51a449d7171fcc1ad277b16d37be4e03e23f33f..80c204fa10e963fa264aa8f64d116cfc5ee1277f 100644 --- a/tests/script/windows/vector/metrics_tag.sim +++ b/tests/script/windows/vector/metrics_tag.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/metrics_time.sim b/tests/script/windows/vector/metrics_time.sim index 716f49d1e5ffeff106914626ac293e8cf0cc0c9e..c127fe78fc1acf712cc5505a680477c0e52d8514 100644 --- a/tests/script/windows/vector/metrics_time.sim +++ b/tests/script/windows/vector/metrics_time.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/multi.sim b/tests/script/windows/vector/multi.sim index 415384d2434ac0f54d3db639421406bc5c854859..ff63cda9a5777a3f20102aafb388f9a7b491a972 100644 --- a/tests/script/windows/vector/multi.sim +++ b/tests/script/windows/vector/multi.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/single.sim b/tests/script/windows/vector/single.sim index f3f3862e54291fcd9cbeddcc684f94c055d78b19..fb3a52760b4efb5133998a9d71d0b7ce2957e69b 100644 --- a/tests/script/windows/vector/single.sim +++ b/tests/script/windows/vector/single.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/table_field.sim b/tests/script/windows/vector/table_field.sim index 0c5df695fb25a6e0f13a5f3e309ad5b203b94962..10c5148243793adbd01221743cbda673ae7ce651 100644 --- a/tests/script/windows/vector/table_field.sim +++ b/tests/script/windows/vector/table_field.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/table_mix.sim b/tests/script/windows/vector/table_mix.sim index 3d660b5611e770ce99a934e11847dd23584387b5..7418cb453df7fc627acc9f34598db6ce7db144d9 100644 --- a/tests/script/windows/vector/table_mix.sim +++ b/tests/script/windows/vector/table_mix.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/table_query.sim b/tests/script/windows/vector/table_query.sim index 27fcd0f65450380411a7f18c5db2099c6d8c93a3..7654688b269eb16c278c000d9da565515512062a 100644 --- a/tests/script/windows/vector/table_query.sim +++ b/tests/script/windows/vector/table_query.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/windows/vector/table_time.sim b/tests/script/windows/vector/table_time.sim index 8a3804c6196375c84a2a49e19f3b42c8d7b57ec8..bea9d41d1bcdfa77acd5eaf52a2de22353b075d9 100644 --- a/tests/script/windows/vector/table_time.sim +++ b/tests/script/windows/vector/table_time.sim @@ -1,5 +1,5 @@ sql connect -sleep 3000 +sleep 2000 sql show databases sql drop database $data00 -x e1 diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 6cdd63b42d28437dded9dc7f99a3888b232c16fc..9ed58a90d1ba6bd781d306070e03f56b75984093 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -1,29 +1,30 @@ @echo off echo TDengine in windows -echo Start TDengine Testing Case ... +rem echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" -echo SCRIPT_DIR: %SCRIPT_DIR% +rem echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" -set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" -echo BUILD_DIR: %BUILD_DIR% +set "BUILD_DIR=%SCRIPT_DIR%..\..\..\debug\build\bin\" +set "TSIM=%BUILD_DIR%tsim" +rem echo BUILD_DIR: %BUILD_DIR% +rem echo TSIM: %TSIM% -set "SIM_DIR=%~dp0..\..\sim" -echo SIM_DIR: %SIM_DIR% +set "SIM_DIR=%SCRIPT_DIR%..\..\..\sim\" +rem echo SIM_DIR: %SIM_DIR% -set "TSIM_DIR=%~dp0..\..\sim\tsim" -echo TSIM_DIR: %TSIM_DIR% +set "TSIM_DIR=%SIM_DIR%tsim\" +rem echo TSIM_DIR: %TSIM_DIR% -set "CFG_DIR=%~dp0..\..\sim\tsim\cfg" -echo CFG_DIR: %CFG_DIR% +set "CFG_DIR=%TSIM_DIR%cfg\" +rem echo CFG_DIR: %CFG_DIR% -set "LOG_DIR=%~dp0..\..\sim\tsim\log" -echo LOG_DIR: %LOG_DIR% +set "LOG_DIR=%TSIM_DIR%log\" +rem echo LOG_DIR: %LOG_DIR% -set "TAOS_CFG=%~dp0..\..\sim\tsim\cfg\taos.cfg" -echo TAOS_CFG: %TAOS_CFG% +set "TAOS_CFG=%CFG_DIR%taos.cfg" +rem echo TAOS_CFG: %TAOS_CFG% if not exist %SIM_DIR% mkdir %SIM_DIR% if not exist %TSIM_DIR% mkdir %TSIM_DIR% @@ -32,29 +33,30 @@ if exist %LOG_DIR% rmdir /s/q %LOG_DIR% if not exist %CFG_DIR% mkdir %CFG_DIR% if not exist %LOG_DIR% mkdir %LOG_DIR% -echo firstEp %FIRSTEP% > %TAOS_CFG% -echo serverPort 6030 >> %TAOS_CFG% -echo wal 2 >> %TAOS_CFG% -echo asyncLog 0 >> %TAOS_CFG% -echo locale en_US.UTF-8 >> %TAOS_CFG% -echo logDir %LOG_DIR% >> %TAOS_CFG% -echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% -echo numOfLogLines 100000000 >> %TAOS_CFG% -echo tmrDebugFlag 131 >> %TAOS_CFG% -echo rpcDebugFlag 143 >> %TAOS_CFG% -echo cDebugFlag 143 >> %TAOS_CFG% -echo qdebugFlag 143 >> %TAOS_CFG% -echo udebugFlag 143 >> %TAOS_CFG% - -set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=192.168.1.182" +rem set "fqdn=" +for /f "skip=1" %%A in ( + 'wmic computersystem get caption' +) do if not defined fqdn set "fqdn=%%A" + +echo firstEp %fqdn% > %TAOS_CFG% +echo fqdn %fqdn% >> %TAOS_CFG% +echo serverPort 7100 >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% +echo numOfLogLines 100000000 >> %TAOS_CFG% +echo rpcDebugFlag 135 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo cDebugFlag 135 >> %TAOS_CFG% +echo udebugFlag 135 >> %TAOS_CFG% +echo wal 0 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo enableCoreFile 1 >> %TAOS_CFG% + +set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" -if "%1" == "-h" set "FIRSTEP=%2" -if "%3" == "-f" set "FILE_NAME=%4" -if "%3" == "-h" set "FIRSTEP=%4" -echo FILE_NAME: %FILE_NAME% -echo FIRSTEP: %FIRSTEP% +rem echo FILE_NAME: %FILE_NAME% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% -%tsim% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file +%TSIM% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file diff --git a/tests/test-all.sh b/tests/test-all.sh index 0c1f55f5f019b102fb35ad800de7627a72cc2d9a..3c8aed7d186aab0038103fe50d57137d79db08e2 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -6,25 +6,71 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' -function git_branch { - branch="`git branch 2>/dev/null | grep "^\*" | sed -e "s/^\*\ //"`" - if [ "${branch}" != "" ];then - if [ "${branch}" = "(no branch)" ];then - branch="(`git rev-parse --short HEAD`...)" - fi - branch=(${branch////_}) - echo "$branch" - fi + +tests_dir=`pwd` +IN_TDINTERNAL="community" + +function stopTaosd { + echo "Stop taosd" + sudo systemctl stop taosd + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +function dohavecore(){ + corefile=`find $corepath -mmin 1` + core_file=`echo $corefile|cut -d " " -f2` + proc=`echo $corefile|cut -d "_" -f3` + if [ -n "$corefile" ];then + echo 'taosd or taos has generated core' + rm case.log + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]] && [[ $1 == 1 ]]; then + cd ../../../ + tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* + if [[ $2 == 1 ]];then + cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` + else + cd community + cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + fi + else + cd ../../ + if [[ $1 == 1 ]];then + tar -zcPf $corepath'taos_'`date "+%Y_%m_%d_%H_%M_%S"`.tar.gz debug/build/bin/taosd debug/build/bin/tsim debug/build/lib/libtaos*so* + cp -r sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + fi + fi + if [[ $1 == 1 ]];then + echo '\n'|gdb /usr/local/taos/bin/$proc $core_file -ex "bt 10" -ex quit + exit 8 + fi + fi } + function runSimCaseOneByOne { while read -r line; do if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then - case=`echo $line | grep sim$ |awk '{print $NF}'` - - start_time=`date +%s` - ./test.sh -f $case > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ - echo -e "${RED}$case failed${NC}" | tee -a out.log + case=`echo $line | grep sim$ |awk '{print $NF}'` + start_time=`date +%s` + date +%F\ %T | tee -a out.log + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + echo -n $case + ./test.sh -f $case > /dev/null 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + echo -e "${RED} failed${NC}" | tee -a out.log + else + echo -n $case + ./test.sh -f $case > /dev/null 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + echo -e "${RED} failed${NC}" | tee -a out.log + fi out_log=`tail -1 out.log ` # if [[ $out_log =~ 'failed' ]];then # exit 8 @@ -35,35 +81,50 @@ function runSimCaseOneByOne { done < $1 } function runSimCaseOneByOnefq { - while read -r line; do + + start=`sed -n "/$1-start/=" jenkins/basic.txt` + end=`sed -n "/$1-end/=" jenkins/basic.txt` + for ((i=$start;i<=$end;i++)) ; do + line=`sed -n "$i"p jenkins/basic.txt` if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then case=`echo $line | grep sim$ |awk '{print $NF}'` - start_time=`date +%s` - IN_TDINTERNAL="community" + start_time=`date +%s` + date +%F\ %T | tee -a out.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then - ./test.sh -f $case > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ - ( grep 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log + echo -n $case + ./test.sh -f $case > case.log 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) else - ./test.sh -f $case > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ - ( grep 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log + echo -n $case + ./test.sh -f $case > ../../sim/case.log 2>&1 && \ + ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ + ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) fi out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then + rm case.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then - cp -r ../../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S"` + cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` else - cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` + cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + fi + dohavecore $2 1 + if [[ $2 == 1 ]];then + exit 8 fi - exit 8 fi end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log + dohavecore $2 1 fi - done < $1 + done + rm -rf ../../../sim/case.log + rm -rf ../../sim/case.log } function runPyCaseOneByOne { @@ -77,9 +138,11 @@ function runPyCaseOneByOne { case=`echo $line|awk '{print $NF}'` fi start_time=`date +%s` + date +%F\ %T | tee -a pytest-out.log + echo -n $case $line > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \ - echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log + echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ + echo -e "${RED} failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` # if [[ $out_log =~ 'failed' ]];then @@ -92,8 +155,17 @@ function runPyCaseOneByOne { fi done < $1 } -function runPyCaseOneByOnefq { - while read -r line; do +function runPyCaseOneByOnefq() { + cd $tests_dir/pytest + if [[ $1 =~ full ]] ; then + start=1 + end=`sed -n '$=' fulltest.sh` + else + start=`sed -n "/$1-start/=" fulltest.sh` + end=`sed -n "/$1-end/=" fulltest.sh` + fi + for ((i=$start;i<=$end;i++)) ; do + line=`sed -n "$i"p fulltest.sh` if [[ $line =~ ^python.* ]]; then if [[ $line != *sleep* ]]; then @@ -103,28 +175,41 @@ function runPyCaseOneByOnefq { case=`echo $line|awk '{print $NF}'` fi start_time=`date +%s` - $line > /dev/null 2>&1 && \ - echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \ - echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log + date +%F\ %T | tee -a pytest-out.log + echo -n $case + $line > case.log 2>&1 && \ + echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ + echo -e "${RED} failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` if [[ $out_log =~ 'failed' ]];then - cp -r ../../sim ~/sim_$(git_branch)_`date "+%Y_%m_%d_%H:%M:%S" ` - exit 8 + cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` + echo '=====================log===================== ' + cat case.log + rm -rf case.log + dohavecore $2 2 + if [[ $2 == 1 ]];then + exit 8 + fi fi echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log else $line > /dev/null 2>&1 fi + dohavecore $2 2 fi - done < $1 + done + rm -rf ../../sim/case.log } + totalFailed=0 totalPyFailed=0 +totalJDBCFailed=0 +totalUnitFailed=0 +totalExampleFailed=0 -tests_dir=`pwd` - -if [ "$2" != "python" ]; then +corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern` +if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != "example" ]; then echo "### run TSIM test case ###" cd $tests_dir/script @@ -137,26 +222,38 @@ if [ "$2" != "python" ]; then runSimCaseOneByOne jenkins/basic.txt elif [ "$1" == "b1" ]; then echo "### run TSIM b1 test ###" - runSimCaseOneByOne jenkins/basic_1.txt - runSimCaseOneByOne jenkins/basic_4.txt + runSimCaseOneByOnefq b1 0 + runSimCaseOneByOnefq b4 0 + runSimCaseOneByOnefq b7 0 elif [ "$1" == "b2" ]; then echo "### run TSIM b2 test ###" - runSimCaseOneByOne jenkins/basic_2.txt + runSimCaseOneByOnefq b2 0 + runSimCaseOneByOnefq b5 0 elif [ "$1" == "b3" ]; then echo "### run TSIM b3 test ###" - runSimCaseOneByOne jenkins/basic_3.txt + runSimCaseOneByOnefq b3 0 + runSimCaseOneByOnefq b6 0 elif [ "$1" == "b1fq" ]; then echo "### run TSIM b1 test ###" - runSimCaseOneByOnefq jenkins/basic_1.txt + runSimCaseOneByOnefq b1 1 elif [ "$1" == "b2fq" ]; then echo "### run TSIM b2 test ###" - runSimCaseOneByOnefq jenkins/basic_2.txt + runSimCaseOneByOnefq b2 1 elif [ "$1" == "b3fq" ]; then echo "### run TSIM b3 test ###" - runSimCaseOneByOnefq jenkins/basic_3.txt + runSimCaseOneByOnefq b3 1 elif [ "$1" == "b4fq" ]; then echo "### run TSIM b4 test ###" - runSimCaseOneByOnefq jenkins/basic_4.txt + runSimCaseOneByOnefq b4 1 + elif [ "$1" == "b5fq" ]; then + echo "### run TSIM b5 test ###" + runSimCaseOneByOnefq b5 1 + elif [ "$1" == "b6fq" ]; then + echo "### run TSIM b6 test ###" + runSimCaseOneByOnefq b6 1 + elif [ "$1" == "b7fq" ]; then + echo "### run TSIM b7 test ###" + runSimCaseOneByOnefq b7 1 elif [ "$1" == "smoke" ] || [ -z "$1" ]; then echo "### run TSIM smoke test ###" runSimCaseOneByOne basicSuite.sim @@ -181,11 +278,10 @@ if [ "$2" != "python" ]; then fi fi -if [ "$2" != "sim" ]; then +if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$2" != "example" ]; then echo "### run Python test case ###" cd $tests_dir - IN_TDINTERNAL="community" if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then cd ../.. @@ -218,13 +314,19 @@ if [ "$2" != "sim" ]; then runPyCaseOneByOne fulltest.sh elif [ "$1" == "pytestfq" ]; then echo "### run Python full test ###" - runPyCaseOneByOnefq fulltest.sh + runPyCaseOneByOnefq full 0 elif [ "$1" == "p1" ]; then echo "### run Python_1 test ###" - runPyCaseOneByOnefq pytest_1.sh + runPyCaseOneByOnefq p1 1 elif [ "$1" == "p2" ]; then echo "### run Python_2 test ###" - runPyCaseOneByOnefq pytest_2.sh + runPyCaseOneByOnefq p2 1 + elif [ "$1" == "p3" ]; then + echo "### run Python_3 test ###" + runPyCaseOneByOnefq p3 1 + elif [ "$1" == "p4" ]; then + echo "### run Python_4 test ###" + runPyCaseOneByOnefq p4 1 elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then exit $(($totalFailed + $totalPyFailed)) elif [ "$1" == "smoke" ] || [ -z "$1" ]; then @@ -244,4 +346,161 @@ if [ "$2" != "sim" ]; then fi fi -exit $(($totalFailed + $totalPyFailed)) + +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" != "example" ] && [ "$1" == "full" ]; then + echo "### run JDBC test cases ###" + + cd $tests_dir + + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../../ + else + cd ../ + fi + + pwd + cd debug/ + + stopTaosd + rm -rf /var/lib/taos/* + nohup build/bin/taosd -c /etc/taos/ > /dev/null 2>&1 & + sleep 30 + + cd $tests_dir/../src/connector/jdbc + + mvn test > jdbc-out.log 2>&1 + tail -n 20 jdbc-out.log + + cases=`grep 'Tests run' jdbc-out.log | awk 'END{print $3}'` + totalJDBCCases=`echo ${cases/%,}` + failed=`grep 'Tests run' jdbc-out.log | awk 'END{print $5}'` + JDBCFailed=`echo ${failed/%,}` + error=`grep 'Tests run' jdbc-out.log | awk 'END{print $7}'` + JDBCError=`echo ${error/%,}` + + totalJDBCFailed=`expr $JDBCFailed + $JDBCError` + totalJDBCSuccess=`expr $totalJDBCCases - $totalJDBCFailed` + + if [ "$totalJDBCSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalJDBCSuccess JDBC case(s) succeed! ### ${NC}" + fi + + if [ "$totalJDBCFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalJDBCFailed JDBC case(s) failed! ### ${NC}" + fi + dohavecore 1 +fi + +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "example" ] && [ "$1" == "full" ]; then + echo "### run Unit tests ###" + + stopTaosd + cd $tests_dir + + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../../ + else + cd ../ + fi + + pwd + cd debug/build/bin + rm -rf /var/lib/taos/* + nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & + sleep 30 + + pwd + ./queryTest > unittest-out.log 2>&1 + tail -n 20 unittest-out.log + + totalUnitTests=`grep "Running" unittest-out.log | awk '{print $3}'` + totalUnitSuccess=`grep 'PASSED' unittest-out.log | awk '{print $4}'` + totalUnitFailed=`expr $totalUnitTests - $totalUnitSuccess` + + if [ "$totalUnitSuccess" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalUnitSuccess Unit test succeed! ### ${NC}" + fi + + if [ "$totalUnitFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalUnitFailed Unit test failed! ### ${NC}" + fi + dohavecore 1 +fi + +if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$1" == "full" ]; then + echo "### run Example tests ###" + + stopTaosd + cd $tests_dir + + if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then + cd ../../ + else + cd ../ + fi + + pwd + cd debug/build/bin + rm -rf /var/lib/taos/* + nohup ./taosd -c /etc/taos/ > /dev/null 2>&1 & + echo "sleeping for 30 seconds" + #sleep 30 + + cd $tests_dir + echo "current dir: " + pwd + cd examples/c + echo "building applications" + make > /dev/null + totalExamplePass=0 + + echo "Running tests" + ./apitest > /dev/null 2>&1 + if [ $? != "0" ]; then + echo "apitest failed" + totalExampleFailed=`expr $totalExampleFailed + 1` + else + echo "apitest pass" + totalExamplePass=`expr $totalExamplePass + 1` + fi + + ./prepare 127.0.0.1 > /dev/null 2>&1 + if [ $? != "0" ]; then + echo "prepare failed" + totalExampleFailed=`expr $totalExampleFailed + 1` + else + echo "prepare pass" + totalExamplePass=`expr $totalExamplePass + 1` + fi + + ./subscribe -test > /dev/null 2>&1 + if [ $? != "0" ]; then + echo "subscribe failed" + totalExampleFailed=`expr $totalExampleFailed + 1` + else + echo "subscribe pass" + totalExamplePass=`expr $totalExamplePass + 1` + fi + + yes |./asyncdemo 127.0.0.1 test 1000 10 > /dev/null 2>&1 + if [ $? != "0" ]; then + echo "asyncdemo failed" + totalExampleFailed=`expr $totalExampleFailed + 1` + else + echo "asyncdemo pass" + totalExamplePass=`expr $totalExamplePass + 1` + fi + + if [ "$totalExamplePass" -gt "0" ]; then + echo -e "\n${GREEN} ### Total $totalExamplePass examples succeed! ### ${NC}" + fi + + if [ "$totalExampleFailed" -ne "0" ]; then + echo -e "\n${RED} ### Total $totalExampleFailed examples failed! ### ${NC}" + fi + + dohavecore 1 +fi + + +exit $(($totalFailed + $totalPyFailed + $totalJDBCFailed + $totalUnitFailed + $totalExampleFailed)) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 33e1528b70b5525da23723dc9bf6309870d172ee..2eb8ee1614b286f3827705865cf073a7eded0c88 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -1,4 +1,4 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 3.5) +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index a23dff13be07c1a56da6d3051a4384ccfb96d9bb..dbda5f1bbdd01a451376f8afe7717b7290980796 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -146,7 +146,7 @@ char *simGetVariable(SScript *script, char *varName, int32_t varLen) { int32_t simExecuteExpression(SScript *script, char *exp) { char * op1, *op2, *var1, *var2, *var3, *rest; int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; - char t0[512], t1[512], t2[512], t3[1024]; + char t0[1024], t1[1024], t2[1024], t3[2048]; int32_t result; rest = paGetToken(exp, &var1, &var1Len); @@ -301,11 +301,37 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { return true; } +void simReplaceShToBat(char *dst) { + char* sh = strstr(dst, ".sh"); + if (sh != NULL) { + int32_t dstLen = (int32_t)strlen(dst); + char *end = dst + dstLen; + *(end + 1) = 0; + + for (char *p = end; p >= sh; p--) { + *(p + 1) = *p; + } + + sh[0] = '.'; + sh[1] = 'b'; + sh[2] = 'a'; + sh[3] = 't'; + sh[4] = ' '; + } + + simDebug("system cmd is %s", dst); +} + bool simExecuteSystemCmd(SScript *script, char *option) { char buf[4096] = {0}; +#ifndef WINDOWS sprintf(buf, "cd %s; ", tsScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); +#else + sprintf(buf, "%s%s", tsScriptDir, option); + simReplaceShToBat(buf); +#endif simLogSql(buf, true); int32_t code = system(buf); @@ -314,9 +340,7 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, errno, strerror(errno), repeatTimes); taosMsleep(1000); -#ifdef LINUX - signal(SIGCHLD, SIG_DFL); -#endif + taosDflSignal(SIGCHLD); if (repeatTimes++ >= 10) { exit(0); } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 8f13254f68165a9e26ecac2406538f8cba905120..990ea7141335b3971b7a3b4dca5fc6fec6dcd24f 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -20,8 +20,9 @@ #undef TAOS_MEM_CHECK bool simAsyncQuery = false; +bool simExecSuccess = false; -void simHandleSignal(int32_t signo) { +void simHandleSignal(int32_t signo, void *sigInfo, void *context) { simSystemCleanUp(); exit(1); } @@ -51,7 +52,7 @@ int32_t main(int32_t argc, char *argv[]) { } simInfo("simulator is running ..."); - signal(SIGINT, simHandleSignal); + taosSetSignal(SIGINT, simHandleSignal); SScript *script = simParseScript(scriptFile); if (script == NULL) { @@ -62,5 +63,8 @@ int32_t main(int32_t argc, char *argv[]) { simScriptList[++simScriptPos] = script; simExecuteScript(script); - return 0; + int32_t ret = simExecSuccess ? 0 : -1; + simInfo("execute result %d", ret); + + return ret; } \ No newline at end of file diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index bf47c56718b7377aa5cd4fd48269eec0fc9e0b38..40937e70532897fa6776965b861a8d678532d8aa 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -21,6 +21,7 @@ #include "ttimer.h" #include "tutil.h" #include "tsocket.h" +#include "taoserror.h" #undef TAOS_MEM_CHECK SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; @@ -31,6 +32,8 @@ int32_t simDebugFlag = 135; void simCloseTaosdConnect(SScript *script); char simHostName[128]; +extern bool simExecSuccess; + char *simParseArbitratorName(char *varName) { static char hostName[140]; sprintf(hostName, "%s:%d", simHostName, 8000); @@ -78,8 +81,10 @@ char *simParseHostName(char *varName) { } bool simSystemInit() { + if (taos_init()) { + return false; + } taosGetFqdn(simHostName); - taos_init(); simInitsimCmdList(); memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM); return true; @@ -118,10 +123,12 @@ SScript *simProcessCallOver(SScript *script) { if (script->type == SIM_SCRIPT_TYPE_MAIN) { simDebug("script:%s, is main script, set stop flag", script->fileName); if (script->killed) { + simExecSuccess = false; simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s", script->fileName, script->error); return NULL; } else { + simExecSuccess = true; simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX, script->fileName); simCloseTaosdConnect(script);